tesseract  3.05.01
boxread.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: boxread.cpp
3  * Description: Read data from a box file.
4  * Author: Ray Smith
5  * Created: Fri Aug 24 17:47:23 PDT 2007
6  *
7  * (C) Copyright 2007, Google Inc.
8  ** Licensed under the Apache License, Version 2.0 (the "License");
9  ** you may not use this file except in compliance with the License.
10  ** You may obtain a copy of the License at
11  ** http://www.apache.org/licenses/LICENSE-2.0
12  ** Unless required by applicable law or agreed to in writing, software
13  ** distributed under the License is distributed on an "AS IS" BASIS,
14  ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  ** See the License for the specific language governing permissions and
16  ** limitations under the License.
17  *
18  **********************************************************************/
19 
20 #include "boxread.h"
21 #include <string.h>
22 
23 #include "fileerr.h"
24 #include "rect.h"
25 #include "strngs.h"
26 #include "tprintf.h"
27 #include "unichar.h"
28 
29 // Special char code used to identify multi-blob labels.
30 static const char* kMultiBlobLabelCode = "WordStr";
31 
32 // Open the boxfile based on the given image filename.
33 FILE* OpenBoxFile(const STRING& fname) {
34  STRING filename = BoxFileName(fname);
35  FILE* box_file = NULL;
36  if (!(box_file = fopen(filename.string(), "rb"))) {
37  CANTOPENFILE.error("read_next_box", TESSEXIT, "Can't open box file %s",
38  filename.string());
39  }
40  return box_file;
41 }
42 
43 // Reads all boxes from the given filename.
44 // Reads a specific target_page number if >= 0, or all pages otherwise.
45 // Skips blanks if skip_blanks is true.
46 // The UTF-8 label of the box is put in texts, and the full box definition as
47 // a string is put in box_texts, with the corresponding page number in pages.
48 // Each of the output vectors is optional (may be NULL).
49 // Returns false if no boxes are found.
50 bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING& filename,
51  GenericVector<TBOX>* boxes,
52  GenericVector<STRING>* texts,
53  GenericVector<STRING>* box_texts,
54  GenericVector<int>* pages) {
55  GenericVector<char> box_data;
57  return false;
58  return ReadMemBoxes(target_page, skip_blanks, &box_data[0], boxes, texts,
59  box_texts, pages);
60 }
61 
62 // Reads all boxes from the string. Otherwise, as ReadAllBoxes.
63 bool ReadMemBoxes(int target_page, bool skip_blanks, const char* box_data,
64  GenericVector<TBOX>* boxes,
65  GenericVector<STRING>* texts,
66  GenericVector<STRING>* box_texts,
67  GenericVector<int>* pages) {
68  STRING box_str(box_data);
70  box_str.split('\n', &lines);
71  if (lines.empty()) return false;
72  int num_boxes = 0;
73  for (int i = 0; i < lines.size(); ++i) {
74  int page = 0;
75  STRING utf8_str;
76  TBOX box;
77  if (!ParseBoxFileStr(lines[i].string(), &page, &utf8_str, &box)) {
78  continue;
79  }
80  if (skip_blanks && (utf8_str == " " || utf8_str == "\t")) continue;
81  if (target_page >= 0 && page != target_page) continue;
82  if (boxes != NULL) boxes->push_back(box);
83  if (texts != NULL) texts->push_back(utf8_str);
84  if (box_texts != NULL) {
85  STRING full_text;
86  MakeBoxFileStr(utf8_str.string(), box, target_page, &full_text);
87  box_texts->push_back(full_text);
88  }
89  if (pages != NULL) pages->push_back(page);
90  ++num_boxes;
91  }
92  return num_boxes > 0;
93 }
94 
95 // Returns the box file name corresponding to the given image_filename.
96 STRING BoxFileName(const STRING& image_filename) {
97  STRING box_filename = image_filename;
98  const char *lastdot = strrchr(box_filename.string(), '.');
99  if (lastdot != NULL)
100  box_filename.truncate_at(lastdot - box_filename.string());
101 
102  box_filename += ".box";
103  return box_filename;
104 }
105 
106 // TODO(rays) convert all uses of ReadNextBox to use the new ReadAllBoxes.
107 // Box files are used ONLY DURING TRAINING, but by both processes of
108 // creating tr files with tesseract, and unicharset_extractor.
109 // ReadNextBox factors out the code to interpret a line of a box
110 // file so that applybox and unicharset_extractor interpret the same way.
111 // This function returns the next valid box file utf8 string and coords
112 // and returns true, or false on eof (and closes the file).
113 // It ignores the utf8 file signature ByteOrderMark (U+FEFF=EF BB BF), checks
114 // for valid utf-8 and allows space or tab between fields.
115 // utf8_str is set with the unichar string, and bounding box with the box.
116 // If there are page numbers in the file, it reads them all.
117 bool ReadNextBox(int *line_number, FILE* box_file,
118  STRING* utf8_str, TBOX* bounding_box) {
119  return ReadNextBox(-1, line_number, box_file, utf8_str, bounding_box);
120 }
121 
122 // As ReadNextBox above, but get a specific page number. (0-based)
123 // Use -1 to read any page number. Files without page number all
124 // read as if they are page 0.
125 bool ReadNextBox(int target_page, int *line_number, FILE* box_file,
126  STRING* utf8_str, TBOX* bounding_box) {
127  int page = 0;
128  char buff[kBoxReadBufSize]; // boxfile read buffer
129  char *buffptr = buff;
130 
131  while (fgets(buff, sizeof(buff) - 1, box_file)) {
132  (*line_number)++;
133 
134  buffptr = buff;
135  const unsigned char *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
136  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
137  buffptr += 3; // Skip unicode file designation.
138  // Check for blank lines in box file
139  if (*buffptr == '\n' || *buffptr == '\0') continue;
140  // Skip blank boxes.
141  if (*buffptr == ' ' || *buffptr == '\t') continue;
142  if (*buffptr != '\0') {
143  if (!ParseBoxFileStr(buffptr, &page, utf8_str, bounding_box)) {
144  tprintf("Box file format error on line %i; ignored\n", *line_number);
145  continue;
146  }
147  if (target_page >= 0 && target_page != page)
148  continue; // Not on the appropriate page.
149  return true; // Successfully read a box.
150  }
151  }
152  fclose(box_file);
153  return false; // EOF
154 }
155 
156 // Parses the given box file string into a page_number, utf8_str, and
157 // bounding_box. Returns true on a successful parse.
158 // The box file is assumed to contain box definitions, one per line, of the
159 // following format for blob-level boxes:
160 // <UTF8 str> <left> <bottom> <right> <top> <page id>
161 // and for word/line-level boxes:
162 // WordStr <left> <bottom> <right> <top> <page id> #<space-delimited word str>
163 // See applyybox.cpp for more information.
164 bool ParseBoxFileStr(const char* boxfile_str, int* page_number,
165  STRING* utf8_str, TBOX* bounding_box) {
166  *bounding_box = TBOX(); // Initialize it to empty.
167  *utf8_str = "";
168  char uch[kBoxReadBufSize];
169  const char *buffptr = boxfile_str;
170  // Read the unichar without messing up on Tibetan.
171  // According to issue 253 the utf-8 surrogates 85 and A0 are treated
172  // as whitespace by sscanf, so it is more reliable to just find
173  // ascii space and tab.
174  int uch_len = 0;
175  // Skip unicode file designation, if present.
176  const unsigned char *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
177  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
178  buffptr += 3;
179  // Allow a single blank as the UTF-8 string. Check for empty string and
180  // then blindly eat the first character.
181  if (*buffptr == '\0') return false;
182  do {
183  uch[uch_len++] = *buffptr++;
184  } while (*buffptr != '\0' && *buffptr != ' ' && *buffptr != '\t' &&
185  uch_len < kBoxReadBufSize - 1);
186  uch[uch_len] = '\0';
187  if (*buffptr != '\0') ++buffptr;
188  int x_min, y_min, x_max, y_max;
189  *page_number = 0;
190  int count = sscanf(buffptr, "%d %d %d %d %d",
191  &x_min, &y_min, &x_max, &y_max, page_number);
192  if (count != 5 && count != 4) {
193  tprintf("Bad box coordinates in boxfile string! %s\n", ubuf);
194  return false;
195  }
196  // Test for long space-delimited string label.
197  if (strcmp(uch, kMultiBlobLabelCode) == 0 &&
198  (buffptr = strchr(buffptr, '#')) != NULL) {
199  strncpy(uch, buffptr + 1, kBoxReadBufSize - 1);
200  uch[kBoxReadBufSize - 1] = '\0'; // Prevent buffer overrun.
201  chomp_string(uch);
202  uch_len = strlen(uch);
203  }
204  // Validate UTF8 by making unichars with it.
205  int used = 0;
206  while (used < uch_len) {
207  UNICHAR ch(uch + used, uch_len - used);
208  int new_used = ch.utf8_len();
209  if (new_used == 0) {
210  tprintf("Bad UTF-8 str %s starts with 0x%02x at col %d\n",
211  uch + used, uch[used], used + 1);
212  return false;
213  }
214  used += new_used;
215  }
216  *utf8_str = uch;
217  if (x_min > x_max) Swap(&x_min, &x_max);
218  if (y_min > y_max) Swap(&y_min, &y_max);
219  bounding_box->set_to_given_coords(x_min, y_min, x_max, y_max);
220  return true; // Successfully read a box.
221 }
222 
223 // Creates a box file string from a unichar string, TBOX and page number.
224 void MakeBoxFileStr(const char* unichar_str, const TBOX& box, int page_num,
225  STRING* box_str) {
226  *box_str = unichar_str;
227  box_str->add_str_int(" ", box.left());
228  box_str->add_str_int(" ", box.bottom());
229  box_str->add_str_int(" ", box.right());
230  box_str->add_str_int(" ", box.top());
231  box_str->add_str_int(" ", page_num);
232 }
233 
const int kBoxReadBufSize
Definition: boxread.h:31
int count(LIST var_list)
Definition: oldlist.cpp:103
bool ReadNextBox(int *line_number, FILE *box_file, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:117
void split(const char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:289
void set_to_given_coords(int x_min, int y_min, int x_max, int y_max)
Definition: rect.h:263
int push_back(T object)
void add_str_int(const char *str, int number)
Definition: strngs.cpp:384
inT16 bottom() const
Definition: rect.h:61
void truncate_at(inT32 index)
Definition: strngs.cpp:272
const char * string() const
Definition: strngs.cpp:201
bool LoadDataFromFile(const STRING &filename, GenericVector< char > *data)
STRING BoxFileName(const STRING &image_filename)
Definition: boxread.cpp:96
inT16 left() const
Definition: rect.h:68
int utf8_len() const
Definition: unichar.h:72
const ERRCODE CANTOPENFILE
Definition: fileerr.h:25
void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, STRING *box_str)
Definition: boxread.cpp:224
void error(const char *caller, TessErrorLogCode action, const char *format,...) const
Definition: errcode.cpp:40
bool ParseBoxFileStr(const char *boxfile_str, int *page_number, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:164
#define tprintf(...)
Definition: tprintf.h:31
Definition: strngs.h:44
int size() const
Definition: genericvector.h:72
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:63
inT16 top() const
Definition: rect.h:54
Definition: rect.h:30
inT16 right() const
Definition: rect.h:75
FILE * OpenBoxFile(const STRING &fname)
Definition: boxread.cpp:33
bool empty() const
Definition: genericvector.h:84
void chomp_string(char *str)
Definition: helpers.h:75
bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING &filename, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:50
void Swap(T *p1, T *p2)
Definition: helpers.h:90