strgen.cpp

Go to the documentation of this file.
00001 /* $Id: strgen.cpp 25204 2013-04-24 17:54:43Z rubidium $ */
00002 
00003 /*
00004  * This file is part of OpenTTD.
00005  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
00006  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
00007  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
00008  */
00009 
00012 #include "../stdafx.h"
00013 #include "../core/endian_func.hpp"
00014 #include "../string_func.h"
00015 #include "../strings_type.h"
00016 #include "../misc/getoptdata.h"
00017 #include "../table/control_codes.h"
00018 
00019 #include "strgen.h"
00020 
00021 #include <stdarg.h>
00022 #include <exception>
00023 
00024 #if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)
00025 #include <unistd.h>
00026 #include <sys/stat.h>
00027 #endif
00028 
00029 #if defined WIN32 || defined __WATCOMC__
00030 #include <direct.h>
00031 #endif /* WIN32 || __WATCOMC__ */
00032 
00033 #ifdef __MORPHOS__
00034 #ifdef stderr
00035 #undef stderr
00036 #endif
00037 #define stderr stdout
00038 #endif /* __MORPHOS__ */
00039 
00040 #include "../table/strgen_tables.h"
00041 
00042 
00043 #ifdef _MSC_VER
00044 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
00045 #else
00046 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
00047 #endif
00048 
00049 void CDECL strgen_warning(const char *s, ...)
00050 {
00051   char buf[1024];
00052   va_list va;
00053   va_start(va, s);
00054   vsnprintf(buf, lengthof(buf), s, va);
00055   va_end(va);
00056   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
00057   _warnings++;
00058 }
00059 
00060 void CDECL strgen_error(const char *s, ...)
00061 {
00062   char buf[1024];
00063   va_list va;
00064   va_start(va, s);
00065   vsnprintf(buf, lengthof(buf), s, va);
00066   va_end(va);
00067   fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
00068   _errors++;
00069 }
00070 
00071 void NORETURN CDECL strgen_fatal(const char *s, ...)
00072 {
00073   char buf[1024];
00074   va_list va;
00075   va_start(va, s);
00076   vsnprintf(buf, lengthof(buf), s, va);
00077   va_end(va);
00078   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
00079 #ifdef _MSC_VER
00080   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
00081 #endif
00082   throw std::exception();
00083 }
00084 
00085 void NORETURN CDECL error(const char *s, ...)
00086 {
00087   char buf[1024];
00088   va_list va;
00089   va_start(va, s);
00090   vsnprintf(buf, lengthof(buf), s, va);
00091   va_end(va);
00092   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
00093 #ifdef _MSC_VER
00094   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
00095 #endif
00096   exit(2);
00097 }
00098 
00100 struct FileStringReader : StringReader {
00101   FILE *fh; 
00102 
00110   FileStringReader(StringData &data, const char *file, bool master, bool translation) :
00111       StringReader(data, file, master, translation)
00112   {
00113     this->fh = fopen(file, "rb");
00114     if (this->fh == NULL) error("Could not open %s", file);
00115   }
00116 
00118   virtual ~FileStringReader()
00119   {
00120     fclose(this->fh);
00121   }
00122 
00123   /* virtual */ char *ReadLine(char *buffer, size_t size)
00124   {
00125     return fgets(buffer, size, this->fh);
00126   }
00127 
00128   /* virtual */ void HandlePragma(char *str);
00129 
00130   /* virtual */ void ParseFile()
00131   {
00132     this->StringReader::ParseFile();
00133 
00134     if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
00135       error("Language must include ##name, ##ownname and ##isocode");
00136     }
00137   }
00138 };
00139 
00140 void FileStringReader::HandlePragma(char *str)
00141 {
00142   if (!memcmp(str, "id ", 3)) {
00143     this->data.next_string_id = strtoul(str + 3, NULL, 0);
00144   } else if (!memcmp(str, "name ", 5)) {
00145     strecpy(_lang.name, str + 5, lastof(_lang.name));
00146   } else if (!memcmp(str, "ownname ", 8)) {
00147     strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
00148   } else if (!memcmp(str, "isocode ", 8)) {
00149     strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
00150   } else if (!memcmp(str, "textdir ", 8)) {
00151     if (!memcmp(str + 8, "ltr", 3)) {
00152       _lang.text_dir = TD_LTR;
00153     } else if (!memcmp(str + 8, "rtl", 3)) {
00154       _lang.text_dir = TD_RTL;
00155     } else {
00156       error("Invalid textdir %s", str + 8);
00157     }
00158   } else if (!memcmp(str, "digitsep ", 9)) {
00159     str += 9;
00160     strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
00161   } else if (!memcmp(str, "digitsepcur ", 12)) {
00162     str += 12;
00163     strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
00164   } else if (!memcmp(str, "decimalsep ", 11)) {
00165     str += 11;
00166     strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
00167   } else if (!memcmp(str, "winlangid ", 10)) {
00168     const char *buf = str + 10;
00169     long langid = strtol(buf, NULL, 16);
00170     if (langid > (long)UINT16_MAX || langid < 0) {
00171       error("Invalid winlangid %s", buf);
00172     }
00173     _lang.winlangid = (uint16)langid;
00174   } else if (!memcmp(str, "grflangid ", 10)) {
00175     const char *buf = str + 10;
00176     long langid = strtol(buf, NULL, 16);
00177     if (langid >= 0x7F || langid < 0) {
00178       error("Invalid grflangid %s", buf);
00179     }
00180     _lang.newgrflangid = (uint8)langid;
00181   } else if (!memcmp(str, "gender ", 7)) {
00182     if (this->master) error("Genders are not allowed in the base translation.");
00183     char *buf = str + 7;
00184 
00185     for (;;) {
00186       const char *s = ParseWord(&buf);
00187 
00188       if (s == NULL) break;
00189       if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
00190       strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
00191       _lang.num_genders++;
00192     }
00193   } else if (!memcmp(str, "case ", 5)) {
00194     if (this->master) error("Cases are not allowed in the base translation.");
00195     char *buf = str + 5;
00196 
00197     for (;;) {
00198       const char *s = ParseWord(&buf);
00199 
00200       if (s == NULL) break;
00201       if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
00202       strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
00203       _lang.num_cases++;
00204     }
00205   } else {
00206     StringReader::HandlePragma(str);
00207   }
00208 }
00209 
00210 bool CompareFiles(const char *n1, const char *n2)
00211 {
00212   FILE *f2 = fopen(n2, "rb");
00213   if (f2 == NULL) return false;
00214 
00215   FILE *f1 = fopen(n1, "rb");
00216   if (f1 == NULL) error("can't open %s", n1);
00217 
00218   size_t l1, l2;
00219   do {
00220     char b1[4096];
00221     char b2[4096];
00222     l1 = fread(b1, 1, sizeof(b1), f1);
00223     l2 = fread(b2, 1, sizeof(b2), f2);
00224 
00225     if (l1 != l2 || memcmp(b1, b2, l1)) {
00226       fclose(f2);
00227       fclose(f1);
00228       return false;
00229     }
00230   } while (l1 != 0);
00231 
00232   fclose(f2);
00233   fclose(f1);
00234   return true;
00235 }
00236 
00238 struct FileWriter {
00239   FILE *fh;             
00240   const char *filename; 
00241 
00246   FileWriter(const char *filename)
00247   {
00248     this->filename = strdup(filename);
00249     this->fh = fopen(this->filename, "wb");
00250 
00251     if (this->fh == NULL) {
00252       error("Could not open %s", this->filename);
00253     }
00254   }
00255 
00257   void Finalise()
00258   {
00259     fclose(this->fh);
00260     this->fh = NULL;
00261   }
00262 
00264   virtual ~FileWriter()
00265   {
00266     /* If we weren't closed an exception was thrown, so remove the temporary file. */
00267     if (fh != NULL) {
00268       fclose(this->fh);
00269       unlink(this->filename);
00270     }
00271     free(this->filename);
00272   }
00273 };
00274 
00275 struct HeaderFileWriter : HeaderWriter, FileWriter {
00277   const char *real_filename;
00279   int prev;
00280 
00285   HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
00286     real_filename(strdup(filename)), prev(0)
00287   {
00288     fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
00289     fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
00290     fprintf(this->fh, "#define TABLE_STRINGS_H\n");
00291   }
00292 
00293   void WriteStringID(const char *name, int stringid)
00294   {
00295     if (prev + 1 != stringid) fprintf(this->fh, "\n");
00296     fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
00297     prev = stringid;
00298   }
00299 
00300   void Finalise(const StringData &data)
00301   {
00302     /* Find the plural form with the most amount of cases. */
00303     int max_plural_forms = 0;
00304     for (uint i = 0; i < lengthof(_plural_forms); i++) {
00305       max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
00306     }
00307 
00308     fprintf(this->fh,
00309       "\n"
00310       "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
00311       "static const uint LANGUAGE_MAX_PLURAL       = %d;\n"
00312       "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
00313       (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
00314     );
00315 
00316     fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
00317 
00318     this->FileWriter::Finalise();
00319 
00320     if (CompareFiles(this->filename, this->real_filename)) {
00321       /* files are equal. tmp.xxx is not needed */
00322       unlink(this->filename);
00323     } else {
00324       /* else rename tmp.xxx into filename */
00325   #if defined(WIN32) || defined(WIN64)
00326       unlink(this->real_filename);
00327   #endif
00328       if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
00329     }
00330   }
00331 };
00332 
00334 struct LanguageFileWriter : LanguageWriter, FileWriter {
00339   LanguageFileWriter(const char *filename) : FileWriter(filename)
00340   {
00341   }
00342 
00343   void WriteHeader(const LanguagePackHeader *header)
00344   {
00345     this->Write((const byte *)header, sizeof(*header));
00346   }
00347 
00348   void Finalise()
00349   {
00350     fputc(0, this->fh);
00351     this->FileWriter::Finalise();
00352   }
00353 
00354   void Write(const byte *buffer, size_t length)
00355   {
00356     if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
00357       error("Could not write to %s", this->filename);
00358     }
00359   }
00360 };
00361 
00363 static inline void ottd_mkdir(const char *directory)
00364 {
00365 #if defined(WIN32) || defined(__WATCOMC__)
00366     mkdir(directory);
00367 #else
00368     mkdir(directory, 0755);
00369 #endif
00370 }
00371 
00377 static inline char *mkpath(char *buf, size_t buflen, const char *path, const char *file)
00378 {
00379   ttd_strlcpy(buf, path, buflen); // copy directory into buffer
00380 
00381   char *p = strchr(buf, '\0'); // add path separator if necessary
00382   if (p[-1] != PATHSEPCHAR && (size_t)(p - buf) + 1 < buflen) *p++ = PATHSEPCHAR;
00383   ttd_strlcpy(p, file, buflen - (size_t)(p - buf)); // concatenate filename at end of buffer
00384   return buf;
00385 }
00386 
00387 #if defined(__MINGW32__)
00388 
00393 static inline char *replace_pathsep(char *s)
00394 {
00395   for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
00396   return s;
00397 }
00398 #else
00399 static inline char *replace_pathsep(char *s) { return s; }
00400 #endif
00401 
00403 static const OptionData _opts[] = {
00404     GETOPT_NOVAL(     'v',  "--version"),
00405   GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
00406   GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
00407   GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
00408     GETOPT_NOVAL(     't',  "--todo"),
00409     GETOPT_NOVAL(     'w',  "--warning"),
00410     GETOPT_NOVAL(     'h',  "--help"),
00411   GETOPT_GENERAL('h', '?',  NULL,               ODF_NO_VALUE),
00412     GETOPT_VALUE(     's',  "--source_dir"),
00413     GETOPT_VALUE(     'd',  "--dest_dir"),
00414   GETOPT_END(),
00415 };
00416 
00417 int CDECL main(int argc, char *argv[])
00418 {
00419   char pathbuf[MAX_PATH];
00420   const char *src_dir = ".";
00421   const char *dest_dir = NULL;
00422 
00423   GetOptData mgo(argc - 1, argv + 1, _opts);
00424   for (;;) {
00425     int i = mgo.GetOpt();
00426     if (i == -1) break;
00427 
00428     switch (i) {
00429       case 'v':
00430         puts("$Revision: 25204 $");
00431         return 0;
00432 
00433       case 'C':
00434         printf("args\tflags\tcommand\treplacement\n");
00435         for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
00436           char flags;
00437           if (cs->proc == EmitGender) {
00438             flags = 'g'; // Command needs number of parameters defined by number of genders
00439           } else if (cs->proc == EmitPlural) {
00440             flags = 'p'; // Command needs number of parameters defined by plural value
00441           } else if (cs->flags & C_DONTCOUNT) {
00442             flags = 'i'; // Command may be in the translation when it is not in base
00443           } else {
00444             flags = '0'; // Command needs no parameters
00445           }
00446           printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
00447         }
00448         return 0;
00449 
00450       case 'L':
00451         printf("count\tdescription\tnames\n");
00452         for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
00453           printf("%i\t\"%s\"\t%s\n", pf->plural_count, pf->description, pf->names);
00454         }
00455         return 0;
00456 
00457       case 'P':
00458         printf("name\tflags\tdefault\tdescription\n");
00459         for (size_t i = 0; i < lengthof(_pragmas); i++) {
00460           printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
00461               _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
00462         }
00463         return 0;
00464 
00465       case 't':
00466         _show_todo |= 1;
00467         break;
00468 
00469       case 'w':
00470         _show_todo |= 2;
00471         break;
00472 
00473       case 'h':
00474         puts(
00475           "strgen - $Revision: 25204 $\n"
00476           " -v | --version    print version information and exit\n"
00477           " -t | --todo       replace any untranslated strings with '<TODO>'\n"
00478           " -w | --warning    print a warning for any untranslated strings\n"
00479           " -h | -? | --help  print this help message and exit\n"
00480           " -s | --source_dir search for english.txt in the specified directory\n"
00481           " -d | --dest_dir   put output file in the specified directory, create if needed\n"
00482           " -export-commands  export all commands and exit\n"
00483           " -export-plurals   export all plural forms and exit\n"
00484           " -export-pragmas   export all pragmas and exit\n"
00485           " Run without parameters and strgen will search for english.txt and parse it,\n"
00486           " creating strings.h. Passing an argument, strgen will translate that language\n"
00487           " file using english.txt as a reference and output <language>.lng."
00488         );
00489         return 0;
00490 
00491       case 's':
00492         src_dir = replace_pathsep(mgo.opt);
00493         break;
00494 
00495       case 'd':
00496         dest_dir = replace_pathsep(mgo.opt);
00497         break;
00498 
00499       case -2:
00500         fprintf(stderr, "Invalid arguments\n");
00501         return 0;
00502     }
00503   }
00504 
00505   if (dest_dir == NULL) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
00506 
00507   try {
00508     /* strgen has two modes of operation. If no (free) arguments are passed
00509      * strgen generates strings.h to the destination directory. If it is supplied
00510      * with a (free) parameter the program will translate that language to destination
00511      * directory. As input english.txt is parsed from the source directory */
00512     if (mgo.numleft == 0) {
00513       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
00514 
00515       /* parse master file */
00516       StringData data(TAB_COUNT);
00517       FileStringReader master_reader(data, pathbuf, true, false);
00518       master_reader.ParseFile();
00519       if (_errors != 0) return 1;
00520 
00521       /* write strings.h */
00522       ottd_mkdir(dest_dir);
00523       mkpath(pathbuf, lengthof(pathbuf), dest_dir, "strings.h");
00524 
00525       HeaderFileWriter writer(pathbuf);
00526       writer.WriteHeader(data);
00527       writer.Finalise(data);
00528     } else if (mgo.numleft >= 1) {
00529       char *r;
00530 
00531       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
00532 
00533       StringData data(TAB_COUNT);
00534       /* parse master file and check if target file is correct */
00535       FileStringReader master_reader(data, pathbuf, true, false);
00536       master_reader.ParseFile();
00537 
00538       for (int i = 0; i < mgo.numleft; i++) {
00539         data.FreeTranslation();
00540 
00541         const char *translation = replace_pathsep(mgo.argv[i]);
00542         const char *file = strrchr(translation, PATHSEPCHAR);
00543         FileStringReader translation_reader(data, translation, false, file == NULL || strcmp(file + 1, "english.txt") != 0);
00544         translation_reader.ParseFile(); // target file
00545         if (_errors != 0) return 1;
00546 
00547         /* get the targetfile, strip any directories and append to destination path */
00548         r = strrchr(mgo.argv[i], PATHSEPCHAR);
00549         mkpath(pathbuf, lengthof(pathbuf), dest_dir, (r != NULL) ? &r[1] : mgo.argv[i]);
00550 
00551         /* rename the .txt (input-extension) to .lng */
00552         r = strrchr(pathbuf, '.');
00553         if (r == NULL || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
00554         ttd_strlcpy(r, ".lng", (size_t)(r - pathbuf));
00555 
00556         LanguageFileWriter writer(pathbuf);
00557         writer.WriteLang(data);
00558         writer.Finalise();
00559 
00560         /* if showing warnings, print a summary of the language */
00561         if ((_show_todo & 2) != 0) {
00562           fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
00563         }
00564       }
00565     }
00566   } catch (...) {
00567     return 2;
00568   }
00569 
00570   return 0;
00571 }