OpenTTD
fileio.cpp
Go to the documentation of this file.
1 /* $Id: fileio.cpp 27886 2017-06-22 17:29:53Z frosch $ */
2 
3 /*
4  * This file is part of OpenTTD.
5  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
6  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
7  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
8  */
9 
12 #include "stdafx.h"
13 #include "fileio_func.h"
14 #include "debug.h"
15 #include "fios.h"
16 #include "string_func.h"
17 #include "tar_type.h"
18 #ifdef WIN32
19 #include <windows.h>
20 # define access _taccess
21 #elif defined(__HAIKU__)
22 #include <Path.h>
23 #include <storage/FindDirectory.h>
24 #else
25 #include <unistd.h>
26 #include <pwd.h>
27 #endif
28 #include <sys/stat.h>
29 #include <algorithm>
30 
31 #ifdef WITH_XDG_BASEDIR
32 #include "basedir.h"
33 #endif
34 
35 #include "safeguards.h"
36 
38 #define FIO_BUFFER_SIZE 512
39 
41 struct Fio {
42  byte *buffer, *buffer_end;
43  size_t pos;
44  FILE *cur_fh;
45  const char *filename;
48  const char *filenames[MAX_FILE_SLOTS];
50 #if defined(LIMITED_FDS)
51  uint open_handles;
52  uint usage_count[MAX_FILE_SLOTS];
53 #endif /* LIMITED_FDS */
54 };
55 
56 static Fio _fio;
57 
59 static bool _do_scan_working_directory = true;
60 
61 extern char *_config_file;
62 extern char *_highscore_file;
63 
68 size_t FioGetPos()
69 {
70  return _fio.pos + (_fio.buffer - _fio.buffer_end);
71 }
72 
78 const char *FioGetFilename(uint8 slot)
79 {
80  return _fio.shortnames[slot];
81 }
82 
88 void FioSeekTo(size_t pos, int mode)
89 {
90  if (mode == SEEK_CUR) pos += FioGetPos();
91  _fio.buffer = _fio.buffer_end = _fio.buffer_start + FIO_BUFFER_SIZE;
92  _fio.pos = pos;
93  if (fseek(_fio.cur_fh, _fio.pos, SEEK_SET) < 0) {
94  DEBUG(misc, 0, "Seeking in %s failed", _fio.filename);
95  }
96 }
97 
98 #if defined(LIMITED_FDS)
99 static void FioRestoreFile(int slot)
100 {
101  /* Do we still have the file open, or should we reopen it? */
102  if (_fio.handles[slot] == NULL) {
103  DEBUG(misc, 6, "Restoring file '%s' in slot '%d' from disk", _fio.filenames[slot], slot);
104  FioOpenFile(slot, _fio.filenames[slot]);
105  }
106  _fio.usage_count[slot]++;
107 }
108 #endif /* LIMITED_FDS */
109 
115 void FioSeekToFile(uint8 slot, size_t pos)
116 {
117  FILE *f;
118 #if defined(LIMITED_FDS)
119  /* Make sure we have this file open */
120  FioRestoreFile(slot);
121 #endif /* LIMITED_FDS */
122  f = _fio.handles[slot];
123  assert(f != NULL);
124  _fio.cur_fh = f;
125  _fio.filename = _fio.filenames[slot];
126  FioSeekTo(pos, SEEK_SET);
127 }
128 
134 {
135  if (_fio.buffer == _fio.buffer_end) {
136  _fio.buffer = _fio.buffer_start;
137  size_t size = fread(_fio.buffer, 1, FIO_BUFFER_SIZE, _fio.cur_fh);
138  _fio.pos += size;
139  _fio.buffer_end = _fio.buffer_start + size;
140 
141  if (size == 0) return 0;
142  }
143  return *_fio.buffer++;
144 }
145 
150 void FioSkipBytes(int n)
151 {
152  for (;;) {
153  int m = min(_fio.buffer_end - _fio.buffer, n);
154  _fio.buffer += m;
155  n -= m;
156  if (n == 0) break;
157  FioReadByte();
158  n--;
159  }
160 }
161 
166 uint16 FioReadWord()
167 {
168  byte b = FioReadByte();
169  return (FioReadByte() << 8) | b;
170 }
171 
176 uint32 FioReadDword()
177 {
178  uint b = FioReadWord();
179  return (FioReadWord() << 16) | b;
180 }
181 
187 void FioReadBlock(void *ptr, size_t size)
188 {
189  FioSeekTo(FioGetPos(), SEEK_SET);
190  _fio.pos += fread(ptr, 1, size, _fio.cur_fh);
191 }
192 
197 static inline void FioCloseFile(int slot)
198 {
199  if (_fio.handles[slot] != NULL) {
200  fclose(_fio.handles[slot]);
201 
202  free(_fio.shortnames[slot]);
203  _fio.shortnames[slot] = NULL;
204 
205  _fio.handles[slot] = NULL;
206 #if defined(LIMITED_FDS)
207  _fio.open_handles--;
208 #endif /* LIMITED_FDS */
209  }
210 }
211 
214 {
215  for (int i = 0; i != lengthof(_fio.handles); i++) {
216  FioCloseFile(i);
217  }
218 }
219 
220 #if defined(LIMITED_FDS)
221 static void FioFreeHandle()
222 {
223  /* If we are about to open a file that will exceed the limit, close a file */
224  if (_fio.open_handles + 1 == LIMITED_FDS) {
225  uint i, count;
226  int slot;
227 
228  count = UINT_MAX;
229  slot = -1;
230  /* Find the file that is used the least */
231  for (i = 0; i < lengthof(_fio.handles); i++) {
232  if (_fio.handles[i] != NULL && _fio.usage_count[i] < count) {
233  count = _fio.usage_count[i];
234  slot = i;
235  }
236  }
237  assert(slot != -1);
238  DEBUG(misc, 6, "Closing filehandler '%s' in slot '%d' because of fd-limit", _fio.filenames[slot], slot);
239  FioCloseFile(slot);
240  }
241 }
242 #endif /* LIMITED_FDS */
243 
250 void FioOpenFile(int slot, const char *filename, Subdirectory subdir)
251 {
252  FILE *f;
253 
254 #if defined(LIMITED_FDS)
255  FioFreeHandle();
256 #endif /* LIMITED_FDS */
257  f = FioFOpenFile(filename, "rb", subdir);
258  if (f == NULL) usererror("Cannot open file '%s'", filename);
259  long pos = ftell(f);
260  if (pos < 0) usererror("Cannot read file '%s'", filename);
261 
262  FioCloseFile(slot); // if file was opened before, close it
263  _fio.handles[slot] = f;
264  _fio.filenames[slot] = filename;
265 
266  /* Store the filename without path and extension */
267  const char *t = strrchr(filename, PATHSEPCHAR);
268  _fio.shortnames[slot] = stredup(t == NULL ? filename : t);
269  char *t2 = strrchr(_fio.shortnames[slot], '.');
270  if (t2 != NULL) *t2 = '\0';
271  strtolower(_fio.shortnames[slot]);
272 
273 #if defined(LIMITED_FDS)
274  _fio.usage_count[slot] = 0;
275  _fio.open_handles++;
276 #endif /* LIMITED_FDS */
277  FioSeekToFile(slot, (uint32)pos);
278 }
279 
280 static const char * const _subdirs[] = {
281  "",
282  "save" PATHSEP,
283  "save" PATHSEP "autosave" PATHSEP,
284  "scenario" PATHSEP,
285  "scenario" PATHSEP "heightmap" PATHSEP,
286  "gm" PATHSEP,
287  "data" PATHSEP,
288  "baseset" PATHSEP,
289  "newgrf" PATHSEP,
290  "lang" PATHSEP,
291  "ai" PATHSEP,
292  "ai" PATHSEP "library" PATHSEP,
293  "game" PATHSEP,
294  "game" PATHSEP "library" PATHSEP,
295  "screenshot" PATHSEP,
296 };
297 assert_compile(lengthof(_subdirs) == NUM_SUBDIRS);
298 
299 const char *_searchpaths[NUM_SEARCHPATHS];
300 TarList _tar_list[NUM_SUBDIRS];
301 TarFileList _tar_filelist[NUM_SUBDIRS];
302 
303 typedef std::map<std::string, std::string> TarLinkList;
304 static TarLinkList _tar_linklist[NUM_SUBDIRS];
305 
312 bool FioCheckFileExists(const char *filename, Subdirectory subdir)
313 {
314  FILE *f = FioFOpenFile(filename, "rb", subdir);
315  if (f == NULL) return false;
316 
317  FioFCloseFile(f);
318  return true;
319 }
320 
326 bool FileExists(const char *filename)
327 {
328 #if defined(WINCE)
329  /* There is always one platform that doesn't support basic commands... */
330  HANDLE hand = CreateFile(OTTD2FS(filename), 0, 0, NULL, OPEN_EXISTING, 0, NULL);
331  if (hand == INVALID_HANDLE_VALUE) return 1;
332  CloseHandle(hand);
333  return 0;
334 #else
335  return access(OTTD2FS(filename), 0) == 0;
336 #endif
337 }
338 
342 void FioFCloseFile(FILE *f)
343 {
344  fclose(f);
345 }
346 
347 char *FioGetFullPath(char *buf, const char *last, Searchpath sp, Subdirectory subdir, const char *filename)
348 {
349  assert(subdir < NUM_SUBDIRS);
350  assert(sp < NUM_SEARCHPATHS);
351 
352  seprintf(buf, last, "%s%s%s", _searchpaths[sp], _subdirs[subdir], filename);
353  return buf;
354 }
355 
364 char *FioFindFullPath(char *buf, const char *last, Subdirectory subdir, const char *filename)
365 {
366  Searchpath sp;
367  assert(subdir < NUM_SUBDIRS);
368 
369  FOR_ALL_SEARCHPATHS(sp) {
370  FioGetFullPath(buf, last, sp, subdir, filename);
371  if (FileExists(buf)) return buf;
372 #if !defined(WIN32)
373  /* Be, as opening files, aware that sometimes the filename
374  * might be in uppercase when it is in lowercase on the
375  * disk. Of course Windows doesn't care about casing. */
376  if (strtolower(buf + strlen(_searchpaths[sp]) - 1) && FileExists(buf)) return buf;
377 #endif
378  }
379 
380  return NULL;
381 }
382 
383 char *FioAppendDirectory(char *buf, const char *last, Searchpath sp, Subdirectory subdir)
384 {
385  assert(subdir < NUM_SUBDIRS);
386  assert(sp < NUM_SEARCHPATHS);
387 
388  seprintf(buf, last, "%s%s", _searchpaths[sp], _subdirs[subdir]);
389  return buf;
390 }
391 
392 char *FioGetDirectory(char *buf, const char *last, Subdirectory subdir)
393 {
394  Searchpath sp;
395 
396  /* Find and return the first valid directory */
397  FOR_ALL_SEARCHPATHS(sp) {
398  char *ret = FioAppendDirectory(buf, last, sp, subdir);
399  if (FileExists(buf)) return ret;
400  }
401 
402  /* Could not find the directory, fall back to a base path */
403  strecpy(buf, _personal_dir, last);
404 
405  return buf;
406 }
407 
408 static FILE *FioFOpenFileSp(const char *filename, const char *mode, Searchpath sp, Subdirectory subdir, size_t *filesize)
409 {
410 #if defined(WIN32) && defined(UNICODE)
411  /* fopen is implemented as a define with ellipses for
412  * Unicode support (prepend an L). As we are not sending
413  * a string, but a variable, it 'renames' the variable,
414  * so make that variable to makes it compile happily */
415  wchar_t Lmode[5];
416  MultiByteToWideChar(CP_ACP, 0, mode, -1, Lmode, lengthof(Lmode));
417 #endif
418  FILE *f = NULL;
419  char buf[MAX_PATH];
420 
421  if (subdir == NO_DIRECTORY) {
422  strecpy(buf, filename, lastof(buf));
423  } else {
424  seprintf(buf, lastof(buf), "%s%s%s", _searchpaths[sp], _subdirs[subdir], filename);
425  }
426 
427 #if defined(WIN32)
428  if (mode[0] == 'r' && GetFileAttributes(OTTD2FS(buf)) == INVALID_FILE_ATTRIBUTES) return NULL;
429 #endif
430 
431  f = fopen(buf, mode);
432 #if !defined(WIN32)
433  if (f == NULL && strtolower(buf + ((subdir == NO_DIRECTORY) ? 0 : strlen(_searchpaths[sp]) - 1))) {
434  f = fopen(buf, mode);
435  }
436 #endif
437  if (f != NULL && filesize != NULL) {
438  /* Find the size of the file */
439  fseek(f, 0, SEEK_END);
440  *filesize = ftell(f);
441  fseek(f, 0, SEEK_SET);
442  }
443  return f;
444 }
445 
453 FILE *FioFOpenFileTar(TarFileListEntry *entry, size_t *filesize)
454 {
455  FILE *f = fopen(entry->tar_filename, "rb");
456  if (f == NULL) return f;
457 
458  if (fseek(f, entry->position, SEEK_SET) < 0) {
459  fclose(f);
460  return NULL;
461  }
462 
463  if (filesize != NULL) *filesize = entry->size;
464  return f;
465 }
466 
474 FILE *FioFOpenFile(const char *filename, const char *mode, Subdirectory subdir, size_t *filesize)
475 {
476  FILE *f = NULL;
477  Searchpath sp;
478 
479  assert(subdir < NUM_SUBDIRS || subdir == NO_DIRECTORY);
480 
481  FOR_ALL_SEARCHPATHS(sp) {
482  f = FioFOpenFileSp(filename, mode, sp, subdir, filesize);
483  if (f != NULL || subdir == NO_DIRECTORY) break;
484  }
485 
486  /* We can only use .tar in case of data-dir, and read-mode */
487  if (f == NULL && mode[0] == 'r' && subdir != NO_DIRECTORY) {
488  static const uint MAX_RESOLVED_LENGTH = 2 * (100 + 100 + 155) + 1; // Enough space to hold two filenames plus link. See 'TarHeader'.
489  char resolved_name[MAX_RESOLVED_LENGTH];
490 
491  /* Filenames in tars are always forced to be lowercase */
492  strecpy(resolved_name, filename, lastof(resolved_name));
493  strtolower(resolved_name);
494 
495  size_t resolved_len = strlen(resolved_name);
496 
497  /* Resolve ONE directory link */
498  for (TarLinkList::iterator link = _tar_linklist[subdir].begin(); link != _tar_linklist[subdir].end(); link++) {
499  const std::string &src = link->first;
500  size_t len = src.length();
501  if (resolved_len >= len && resolved_name[len - 1] == PATHSEPCHAR && strncmp(src.c_str(), resolved_name, len) == 0) {
502  /* Apply link */
503  char resolved_name2[MAX_RESOLVED_LENGTH];
504  const std::string &dest = link->second;
505  strecpy(resolved_name2, &(resolved_name[len]), lastof(resolved_name2));
506  strecpy(resolved_name, dest.c_str(), lastof(resolved_name));
507  strecpy(&(resolved_name[dest.length()]), resolved_name2, lastof(resolved_name));
508  break; // Only resolve one level
509  }
510  }
511 
512  TarFileList::iterator it = _tar_filelist[subdir].find(resolved_name);
513  if (it != _tar_filelist[subdir].end()) {
514  f = FioFOpenFileTar(&((*it).second), filesize);
515  }
516  }
517 
518  /* Sometimes a full path is given. To support
519  * the 'subdirectory' must be 'removed'. */
520  if (f == NULL && subdir != NO_DIRECTORY) {
521  switch (subdir) {
522  case BASESET_DIR:
523  f = FioFOpenFile(filename, mode, OLD_GM_DIR, filesize);
524  if (f != NULL) break;
525  /* FALL THROUGH */
526  case NEWGRF_DIR:
527  f = FioFOpenFile(filename, mode, OLD_DATA_DIR, filesize);
528  break;
529 
530  default:
531  f = FioFOpenFile(filename, mode, NO_DIRECTORY, filesize);
532  break;
533  }
534  }
535 
536  return f;
537 }
538 
543 static void FioCreateDirectory(const char *name)
544 {
545  /* Ignore directory creation errors; they'll surface later on, and most
546  * of the time they are 'directory already exists' errors anyhow. */
547 #if defined(WIN32) || defined(WINCE)
548  CreateDirectory(OTTD2FS(name), NULL);
549 #elif defined(OS2) && !defined(__INNOTEK_LIBC__)
550  mkdir(OTTD2FS(name));
551 #elif defined(__MORPHOS__) || defined(__AMIGAOS__)
552  char buf[MAX_PATH];
553  strecpy(buf, name, lastof(buf));
554 
555  size_t len = strlen(name) - 1;
556  if (buf[len] == '/') {
557  buf[len] = '\0'; // Kill pathsep, so mkdir() will not fail
558  }
559 
560  mkdir(OTTD2FS(buf), 0755);
561 #else
562  mkdir(OTTD2FS(name), 0755);
563 #endif
564 }
565 
573 bool AppendPathSeparator(char *buf, const char *last)
574 {
575  size_t s = strlen(buf);
576 
577  /* Length of string + path separator + '\0' */
578  if (s != 0 && buf[s - 1] != PATHSEPCHAR) {
579  if (&buf[s] >= last) return false;
580 
581  seprintf(buf + s, last, "%c", PATHSEPCHAR);
582  }
583 
584  return true;
585 }
586 
593 char *BuildWithFullPath(const char *dir)
594 {
595  char *dest = MallocT<char>(MAX_PATH);
596  char *last = dest + MAX_PATH - 1;
597  strecpy(dest, dir, last);
598 
599  /* Check if absolute or relative path */
600  const char *s = strchr(dest, PATHSEPCHAR);
601 
602  /* Add absolute path */
603  if (s == NULL || dest != s) {
604  if (getcwd(dest, MAX_PATH) == NULL) *dest = '\0';
605  AppendPathSeparator(dest, last);
606  strecat(dest, dir, last);
607  }
608  AppendPathSeparator(dest, last);
609 
610  return dest;
611 }
612 
618 const char *FioTarFirstDir(const char *tarname, Subdirectory subdir)
619 {
620  TarList::iterator it = _tar_list[subdir].find(tarname);
621  if (it == _tar_list[subdir].end()) return NULL;
622  return (*it).second.dirname;
623 }
624 
625 static void TarAddLink(const std::string &srcParam, const std::string &destParam, Subdirectory subdir)
626 {
627  std::string src = srcParam;
628  std::string dest = destParam;
629  /* Tar internals assume lowercase */
630  std::transform(src.begin(), src.end(), src.begin(), tolower);
631  std::transform(dest.begin(), dest.end(), dest.begin(), tolower);
632 
633  TarFileList::iterator dest_file = _tar_filelist[subdir].find(dest);
634  if (dest_file != _tar_filelist[subdir].end()) {
635  /* Link to file. Process the link like the destination file. */
636  _tar_filelist[subdir].insert(TarFileList::value_type(src, dest_file->second));
637  } else {
638  /* Destination file not found. Assume 'link to directory'
639  * Append PATHSEPCHAR to 'src' and 'dest' if needed */
640  const std::string src_path = ((*src.rbegin() == PATHSEPCHAR) ? src : src + PATHSEPCHAR);
641  const std::string dst_path = (dest.length() == 0 ? "" : ((*dest.rbegin() == PATHSEPCHAR) ? dest : dest + PATHSEPCHAR));
642  _tar_linklist[subdir].insert(TarLinkList::value_type(src_path, dst_path));
643  }
644 }
645 
646 void FioTarAddLink(const char *src, const char *dest, Subdirectory subdir)
647 {
648  TarAddLink(src, dest, subdir);
649 }
650 
656 static void SimplifyFileName(char *name)
657 {
658  /* Force lowercase */
659  strtolower(name);
660 
661  /* Tar-files always have '/' path-separator, but we want our PATHSEPCHAR */
662 #if (PATHSEPCHAR != '/')
663  for (char *n = name; *n != '\0'; n++) if (*n == '/') *n = PATHSEPCHAR;
664 #endif
665 }
666 
673 {
674  _tar_filelist[sd].clear();
675  _tar_list[sd].clear();
676  uint num = this->Scan(".tar", sd, false);
677  if (sd == BASESET_DIR || sd == NEWGRF_DIR) num += this->Scan(".tar", OLD_DATA_DIR, false);
678  return num;
679 }
680 
681 /* static */ uint TarScanner::DoScan(TarScanner::Mode mode)
682 {
683  DEBUG(misc, 1, "Scanning for tars");
684  TarScanner fs;
685  uint num = 0;
686  if (mode & TarScanner::BASESET) {
687  num += fs.DoScan(BASESET_DIR);
688  }
689  if (mode & TarScanner::NEWGRF) {
690  num += fs.DoScan(NEWGRF_DIR);
691  }
692  if (mode & TarScanner::AI) {
693  num += fs.DoScan(AI_DIR);
694  num += fs.DoScan(AI_LIBRARY_DIR);
695  }
696  if (mode & TarScanner::GAME) {
697  num += fs.DoScan(GAME_DIR);
698  num += fs.DoScan(GAME_LIBRARY_DIR);
699  }
700  if (mode & TarScanner::SCENARIO) {
701  num += fs.DoScan(SCENARIO_DIR);
702  num += fs.DoScan(HEIGHTMAP_DIR);
703  }
704  DEBUG(misc, 1, "Scan complete, found %d files", num);
705  return num;
706 }
707 
714 bool TarScanner::AddFile(Subdirectory sd, const char *filename)
715 {
716  this->subdir = sd;
717  return this->AddFile(filename, 0);
718 }
719 
720 bool TarScanner::AddFile(const char *filename, size_t basepath_length, const char *tar_filename)
721 {
722  /* No tar within tar. */
723  assert(tar_filename == NULL);
724 
725  /* The TAR-header, repeated for every file */
726  struct TarHeader {
727  char name[100];
728  char mode[8];
729  char uid[8];
730  char gid[8];
731  char size[12];
732  char mtime[12];
733  char chksum[8];
734  char typeflag;
735  char linkname[100];
736  char magic[6];
737  char version[2];
738  char uname[32];
739  char gname[32];
740  char devmajor[8];
741  char devminor[8];
742  char prefix[155];
743 
744  char unused[12];
745  };
746 
747  /* Check if we already seen this file */
748  TarList::iterator it = _tar_list[this->subdir].find(filename);
749  if (it != _tar_list[this->subdir].end()) return false;
750 
751  FILE *f = fopen(filename, "rb");
752  /* Although the file has been found there can be
753  * a number of reasons we cannot open the file.
754  * Most common case is when we simply have not
755  * been given read access. */
756  if (f == NULL) return false;
757 
758  const char *dupped_filename = stredup(filename);
759  _tar_list[this->subdir][filename].filename = dupped_filename;
760  _tar_list[this->subdir][filename].dirname = NULL;
761 
762  TarLinkList links;
763 
764  TarHeader th;
765  char buf[sizeof(th.name) + 1], *end;
766  char name[sizeof(th.prefix) + 1 + sizeof(th.name) + 1];
767  char link[sizeof(th.linkname) + 1];
768  char dest[sizeof(th.prefix) + 1 + sizeof(th.name) + 1 + 1 + sizeof(th.linkname) + 1];
769  size_t num = 0, pos = 0;
770 
771  /* Make a char of 512 empty bytes */
772  char empty[512];
773  memset(&empty[0], 0, sizeof(empty));
774 
775  for (;;) { // Note: feof() always returns 'false' after 'fseek()'. Cool, isn't it?
776  size_t num_bytes_read = fread(&th, 1, 512, f);
777  if (num_bytes_read != 512) break;
778  pos += num_bytes_read;
779 
780  /* Check if we have the new tar-format (ustar) or the old one (a lot of zeros after 'link' field) */
781  if (strncmp(th.magic, "ustar", 5) != 0 && memcmp(&th.magic, &empty[0], 512 - offsetof(TarHeader, magic)) != 0) {
782  /* If we have only zeros in the block, it can be an end-of-file indicator */
783  if (memcmp(&th, &empty[0], 512) == 0) continue;
784 
785  DEBUG(misc, 0, "The file '%s' isn't a valid tar-file", filename);
786  fclose(f);
787  return false;
788  }
789 
790  name[0] = '\0';
791 
792  /* The prefix contains the directory-name */
793  if (th.prefix[0] != '\0') {
794  strecpy(name, th.prefix, lastof(name));
795  strecat(name, PATHSEP, lastof(name));
796  }
797 
798  /* Copy the name of the file in a safe way at the end of 'name' */
799  strecat(name, th.name, lastof(name));
800 
801  /* Calculate the size of the file.. for some strange reason this is stored as a string */
802  strecpy(buf, th.size, lastof(buf));
803  size_t skip = strtoul(buf, &end, 8);
804 
805  switch (th.typeflag) {
806  case '\0':
807  case '0': { // regular file
808  /* Ignore empty files */
809  if (skip == 0) break;
810 
811  if (strlen(name) == 0) break;
812 
813  /* Store this entry in the list */
814  TarFileListEntry entry;
815  entry.tar_filename = dupped_filename;
816  entry.size = skip;
817  entry.position = pos;
818 
819  /* Convert to lowercase and our PATHSEPCHAR */
820  SimplifyFileName(name);
821 
822  DEBUG(misc, 6, "Found file in tar: %s (" PRINTF_SIZE " bytes, " PRINTF_SIZE " offset)", name, skip, pos);
823  if (_tar_filelist[this->subdir].insert(TarFileList::value_type(name, entry)).second) num++;
824 
825  break;
826  }
827 
828  case '1': // hard links
829  case '2': { // symbolic links
830  /* Copy the destination of the link in a safe way at the end of 'linkname' */
831  strecpy(link, th.linkname, lastof(link));
832 
833  if (strlen(name) == 0 || strlen(link) == 0) break;
834 
835  /* Convert to lowercase and our PATHSEPCHAR */
836  SimplifyFileName(name);
837  SimplifyFileName(link);
838 
839  /* Only allow relative links */
840  if (link[0] == PATHSEPCHAR) {
841  DEBUG(misc, 1, "Ignoring absolute link in tar: %s -> %s", name, link);
842  break;
843  }
844 
845  /* Process relative path.
846  * Note: The destination of links must not contain any directory-links. */
847  strecpy(dest, name, lastof(dest));
848  char *destpos = strrchr(dest, PATHSEPCHAR);
849  if (destpos == NULL) destpos = dest;
850  *destpos = '\0';
851 
852  char *pos = link;
853  while (*pos != '\0') {
854  char *next = strchr(pos, PATHSEPCHAR);
855  if (next == NULL) {
856  next = pos + strlen(pos);
857  } else {
858  /* Terminate the substring up to the path separator character. */
859  *next++= '\0';
860  }
861 
862  if (strcmp(pos, ".") == 0) {
863  /* Skip '.' (current dir) */
864  } else if (strcmp(pos, "..") == 0) {
865  /* level up */
866  if (dest[0] == '\0') {
867  DEBUG(misc, 1, "Ignoring link pointing outside of data directory: %s -> %s", name, link);
868  break;
869  }
870 
871  /* Truncate 'dest' after last PATHSEPCHAR.
872  * This assumes that the truncated part is a real directory and not a link. */
873  destpos = strrchr(dest, PATHSEPCHAR);
874  if (destpos == NULL) destpos = dest;
875  *destpos = '\0';
876  } else {
877  /* Append at end of 'dest' */
878  if (destpos != dest) destpos = strecpy(destpos, PATHSEP, lastof(dest));
879  destpos = strecpy(destpos, pos, lastof(dest));
880  }
881 
882  if (destpos >= lastof(dest)) {
883  DEBUG(misc, 0, "The length of a link in tar-file '%s' is too large (malformed?)", filename);
884  fclose(f);
885  return false;
886  }
887 
888  pos = next;
889  }
890 
891  /* Store links in temporary list */
892  DEBUG(misc, 6, "Found link in tar: %s -> %s", name, dest);
893  links.insert(TarLinkList::value_type(name, dest));
894 
895  break;
896  }
897 
898  case '5': // directory
899  /* Convert to lowercase and our PATHSEPCHAR */
900  SimplifyFileName(name);
901 
902  /* Store the first directory name we detect */
903  DEBUG(misc, 6, "Found dir in tar: %s", name);
904  if (_tar_list[this->subdir][filename].dirname == NULL) _tar_list[this->subdir][filename].dirname = stredup(name);
905  break;
906 
907  default:
908  /* Ignore other types */
909  break;
910  }
911 
912  /* Skip to the next block.. */
913  skip = Align(skip, 512);
914  if (fseek(f, skip, SEEK_CUR) < 0) {
915  DEBUG(misc, 0, "The file '%s' can't be read as a valid tar-file", filename);
916  fclose(f);
917  return false;
918  }
919  pos += skip;
920  }
921 
922  DEBUG(misc, 1, "Found tar '%s' with " PRINTF_SIZE " new files", filename, num);
923  fclose(f);
924 
925  /* Resolve file links and store directory links.
926  * We restrict usage of links to two cases:
927  * 1) Links to directories:
928  * Both the source path and the destination path must NOT contain any further links.
929  * When resolving files at most one directory link is resolved.
930  * 2) Links to files:
931  * The destination path must NOT contain any links.
932  * The source path may contain one directory link.
933  */
934  for (TarLinkList::iterator link = links.begin(); link != links.end(); link++) {
935  const std::string &src = link->first;
936  const std::string &dest = link->second;
937  TarAddLink(src, dest, this->subdir);
938  }
939 
940  return true;
941 }
942 
950 bool ExtractTar(const char *tar_filename, Subdirectory subdir)
951 {
952  TarList::iterator it = _tar_list[subdir].find(tar_filename);
953  /* We don't know the file. */
954  if (it == _tar_list[subdir].end()) return false;
955 
956  const char *dirname = (*it).second.dirname;
957 
958  /* The file doesn't have a sub directory! */
959  if (dirname == NULL) return false;
960 
961  char filename[MAX_PATH];
962  strecpy(filename, tar_filename, lastof(filename));
963  char *p = strrchr(filename, PATHSEPCHAR);
964  /* The file's path does not have a separator? */
965  if (p == NULL) return false;
966 
967  p++;
968  strecpy(p, dirname, lastof(filename));
969  DEBUG(misc, 8, "Extracting %s to directory %s", tar_filename, filename);
970  FioCreateDirectory(filename);
971 
972  for (TarFileList::iterator it2 = _tar_filelist[subdir].begin(); it2 != _tar_filelist[subdir].end(); it2++) {
973  if (strcmp((*it2).second.tar_filename, tar_filename) != 0) continue;
974 
975  strecpy(p, (*it2).first.c_str(), lastof(filename));
976 
977  DEBUG(misc, 9, " extracting %s", filename);
978 
979  /* First open the file in the .tar. */
980  size_t to_copy = 0;
981  FILE *in = FioFOpenFileTar(&(*it2).second, &to_copy);
982  if (in == NULL) {
983  DEBUG(misc, 6, "Extracting %s failed; could not open %s", filename, tar_filename);
984  return false;
985  }
986 
987  /* Now open the 'output' file. */
988  FILE *out = fopen(filename, "wb");
989  if (out == NULL) {
990  DEBUG(misc, 6, "Extracting %s failed; could not open %s", filename, filename);
991  fclose(in);
992  return false;
993  }
994 
995  /* Now read from the tar and write it into the file. */
996  char buffer[4096];
997  size_t read;
998  for (; to_copy != 0; to_copy -= read) {
999  read = fread(buffer, 1, min(to_copy, lengthof(buffer)), in);
1000  if (read <= 0 || fwrite(buffer, 1, read, out) != read) break;
1001  }
1002 
1003  /* Close everything up. */
1004  fclose(in);
1005  fclose(out);
1006 
1007  if (to_copy != 0) {
1008  DEBUG(misc, 6, "Extracting %s failed; still %i bytes to copy", filename, (int)to_copy);
1009  return false;
1010  }
1011  }
1012 
1013  DEBUG(misc, 9, " extraction successful");
1014  return true;
1015 }
1016 
1017 #if defined(WIN32) || defined(WINCE)
1018 
1023 extern void DetermineBasePaths(const char *exe);
1024 #else /* defined(WIN32) || defined(WINCE) */
1025 
1033 static bool ChangeWorkingDirectoryToExecutable(const char *exe)
1034 {
1035  char tmp[MAX_PATH];
1036  strecpy(tmp, exe, lastof(tmp));
1037 
1038  bool success = false;
1039 #ifdef WITH_COCOA
1040  char *app_bundle = strchr(tmp, '.');
1041  while (app_bundle != NULL && strncasecmp(app_bundle, ".app", 4) != 0) app_bundle = strchr(&app_bundle[1], '.');
1042 
1043  if (app_bundle != NULL) *app_bundle = '\0';
1044 #endif /* WITH_COCOA */
1045  char *s = strrchr(tmp, PATHSEPCHAR);
1046  if (s != NULL) {
1047  *s = '\0';
1048 #if defined(__DJGPP__)
1049  /* If we want to go to the root, we can't use cd C:, but we must use '/' */
1050  if (s > tmp && *(s - 1) == ':') chdir("/");
1051 #endif
1052  if (chdir(tmp) != 0) {
1053  DEBUG(misc, 0, "Directory with the binary does not exist?");
1054  } else {
1055  success = true;
1056  }
1057  }
1058  return success;
1059 }
1060 
1072 {
1073  /* No working directory, so nothing to do. */
1074  if (_searchpaths[SP_WORKING_DIR] == NULL) return false;
1075 
1076  /* Working directory is root, so do nothing. */
1077  if (strcmp(_searchpaths[SP_WORKING_DIR], PATHSEP) == 0) return false;
1078 
1079  /* No personal/home directory, so the working directory won't be that. */
1080  if (_searchpaths[SP_PERSONAL_DIR] == NULL) return true;
1081 
1082  char tmp[MAX_PATH];
1083  seprintf(tmp, lastof(tmp), "%s%s", _searchpaths[SP_WORKING_DIR], PERSONAL_DIR);
1084  AppendPathSeparator(tmp, lastof(tmp));
1085  return strcmp(tmp, _searchpaths[SP_PERSONAL_DIR]) != 0;
1086 }
1087 
1092 void DetermineBasePaths(const char *exe)
1093 {
1094  char tmp[MAX_PATH];
1095 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1096  const char *xdg_data_home = xdgDataHome(NULL);
1097  seprintf(tmp, lastof(tmp), "%s" PATHSEP "%s", xdg_data_home,
1098  PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR);
1099  free(xdg_data_home);
1100 
1101  AppendPathSeparator(tmp, lastof(tmp));
1102  _searchpaths[SP_PERSONAL_DIR_XDG] = stredup(tmp);
1103 #endif
1104 #if defined(__MORPHOS__) || defined(__AMIGA__) || defined(DOS) || defined(OS2) || !defined(WITH_PERSONAL_DIR)
1105  _searchpaths[SP_PERSONAL_DIR] = NULL;
1106 #else
1107 #ifdef __HAIKU__
1108  BPath path;
1109  find_directory(B_USER_SETTINGS_DIRECTORY, &path);
1110  const char *homedir = stredup(path.Path());
1111 #else
1112  /* getenv is highly unsafe; duplicate it as soon as possible,
1113  * or at least before something else touches the environment
1114  * variables in any way. It can also contain all kinds of
1115  * unvalidated data we rather not want internally. */
1116  const char *homedir = getenv("HOME");
1117  if (homedir != NULL) {
1118  homedir = stredup(homedir);
1119  }
1120 
1121  if (homedir == NULL) {
1122  const struct passwd *pw = getpwuid(getuid());
1123  homedir = (pw == NULL) ? NULL : stredup(pw->pw_dir);
1124  }
1125 #endif
1126 
1127  if (homedir != NULL) {
1128  ValidateString(homedir);
1129  seprintf(tmp, lastof(tmp), "%s" PATHSEP "%s", homedir, PERSONAL_DIR);
1130  AppendPathSeparator(tmp, lastof(tmp));
1131 
1132  _searchpaths[SP_PERSONAL_DIR] = stredup(tmp);
1133  free(homedir);
1134  } else {
1135  _searchpaths[SP_PERSONAL_DIR] = NULL;
1136  }
1137 #endif
1138 
1139 #if defined(WITH_SHARED_DIR)
1140  seprintf(tmp, lastof(tmp), "%s", SHARED_DIR);
1141  AppendPathSeparator(tmp, lastof(tmp));
1142  _searchpaths[SP_SHARED_DIR] = stredup(tmp);
1143 #else
1144  _searchpaths[SP_SHARED_DIR] = NULL;
1145 #endif
1146 
1147 #if defined(__MORPHOS__) || defined(__AMIGA__)
1148  _searchpaths[SP_WORKING_DIR] = NULL;
1149 #else
1150  if (getcwd(tmp, MAX_PATH) == NULL) *tmp = '\0';
1151  AppendPathSeparator(tmp, lastof(tmp));
1152  _searchpaths[SP_WORKING_DIR] = stredup(tmp);
1153 #endif
1154 
1156 
1157  /* Change the working directory to that one of the executable */
1159  if (getcwd(tmp, MAX_PATH) == NULL) *tmp = '\0';
1160  AppendPathSeparator(tmp, lastof(tmp));
1161  _searchpaths[SP_BINARY_DIR] = stredup(tmp);
1162  } else {
1163  _searchpaths[SP_BINARY_DIR] = NULL;
1164  }
1165 
1166  if (_searchpaths[SP_WORKING_DIR] != NULL) {
1167  /* Go back to the current working directory. */
1168  if (chdir(_searchpaths[SP_WORKING_DIR]) != 0) {
1169  DEBUG(misc, 0, "Failed to return to working directory!");
1170  }
1171  }
1172 
1173 #if defined(__MORPHOS__) || defined(__AMIGA__) || defined(DOS) || defined(OS2)
1174  _searchpaths[SP_INSTALLATION_DIR] = NULL;
1175 #else
1176  seprintf(tmp, lastof(tmp), "%s", GLOBAL_DATA_DIR);
1177  AppendPathSeparator(tmp, lastof(tmp));
1178  _searchpaths[SP_INSTALLATION_DIR] = stredup(tmp);
1179 #endif
1180 #ifdef WITH_COCOA
1181 extern void cocoaSetApplicationBundleDir();
1182  cocoaSetApplicationBundleDir();
1183 #else
1184  _searchpaths[SP_APPLICATION_BUNDLE_DIR] = NULL;
1185 #endif
1186 }
1187 #endif /* defined(WIN32) || defined(WINCE) */
1188 
1189 const char *_personal_dir;
1190 
1197 void DeterminePaths(const char *exe)
1198 {
1199  DetermineBasePaths(exe);
1200 
1201 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1202  char config_home[MAX_PATH];
1203 
1204  const char *xdg_config_home = xdgConfigHome(NULL);
1205  seprintf(config_home, lastof(config_home), "%s" PATHSEP "%s", xdg_config_home,
1206  PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR);
1207  free(xdg_config_home);
1208 
1209  AppendPathSeparator(config_home, lastof(config_home));
1210 #endif
1211 
1212  Searchpath sp;
1213  FOR_ALL_SEARCHPATHS(sp) {
1214  if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
1215  DEBUG(misc, 4, "%s added as search path", _searchpaths[sp]);
1216  }
1217 
1218  char *config_dir;
1219  if (_config_file != NULL) {
1220  config_dir = stredup(_config_file);
1221  char *end = strrchr(config_dir, PATHSEPCHAR);
1222  if (end == NULL) {
1223  config_dir[0] = '\0';
1224  } else {
1225  end[1] = '\0';
1226  }
1227  } else {
1228  char personal_dir[MAX_PATH];
1229  if (FioFindFullPath(personal_dir, lastof(personal_dir), BASE_DIR, "openttd.cfg") != NULL) {
1230  char *end = strrchr(personal_dir, PATHSEPCHAR);
1231  if (end != NULL) end[1] = '\0';
1232  config_dir = stredup(personal_dir);
1233  _config_file = str_fmt("%sopenttd.cfg", config_dir);
1234  } else {
1235 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1236  /* No previous configuration file found. Use the configuration folder from XDG. */
1237  config_dir = config_home;
1238 #else
1239  static const Searchpath new_openttd_cfg_order[] = {
1241  };
1242 
1243  config_dir = NULL;
1244  for (uint i = 0; i < lengthof(new_openttd_cfg_order); i++) {
1245  if (IsValidSearchPath(new_openttd_cfg_order[i])) {
1246  config_dir = stredup(_searchpaths[new_openttd_cfg_order[i]]);
1247  break;
1248  }
1249  }
1250  assert(config_dir != NULL);
1251 #endif
1252  _config_file = str_fmt("%sopenttd.cfg", config_dir);
1253  }
1254  }
1255 
1256  DEBUG(misc, 3, "%s found as config directory", config_dir);
1257 
1258  _highscore_file = str_fmt("%shs.dat", config_dir);
1259  extern char *_hotkeys_file;
1260  _hotkeys_file = str_fmt("%shotkeys.cfg", config_dir);
1261  extern char *_windows_file;
1262  _windows_file = str_fmt("%swindows.cfg", config_dir);
1263 
1264 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1265  if (config_dir == config_home) {
1266  /* We are using the XDG configuration home for the config file,
1267  * then store the rest in the XDG data home folder. */
1268  _personal_dir = _searchpaths[SP_PERSONAL_DIR_XDG];
1269  FioCreateDirectory(_personal_dir);
1270  } else
1271 #endif
1272  {
1273  _personal_dir = config_dir;
1274  }
1275 
1276  /* Make the necessary folders */
1277 #if !defined(__MORPHOS__) && !defined(__AMIGA__) && defined(WITH_PERSONAL_DIR)
1278  FioCreateDirectory(config_dir);
1279  if (config_dir != _personal_dir) FioCreateDirectory(_personal_dir);
1280 #endif
1281 
1282  DEBUG(misc, 3, "%s found as personal directory", _personal_dir);
1283 
1284  static const Subdirectory default_subdirs[] = {
1286  };
1287 
1288  for (uint i = 0; i < lengthof(default_subdirs); i++) {
1289  char *dir = str_fmt("%s%s", _personal_dir, _subdirs[default_subdirs[i]]);
1290  FioCreateDirectory(dir);
1291  free(dir);
1292  }
1293 
1294  /* If we have network we make a directory for the autodownloading of content */
1295  _searchpaths[SP_AUTODOWNLOAD_DIR] = str_fmt("%s%s", _personal_dir, "content_download" PATHSEP);
1296 #ifdef ENABLE_NETWORK
1297  FioCreateDirectory(_searchpaths[SP_AUTODOWNLOAD_DIR]);
1298 
1299  /* Create the directory for each of the types of content */
1301  for (uint i = 0; i < lengthof(dirs); i++) {
1302  char *tmp = str_fmt("%s%s", _searchpaths[SP_AUTODOWNLOAD_DIR], _subdirs[dirs[i]]);
1303  FioCreateDirectory(tmp);
1304  free(tmp);
1305  }
1306 
1307  extern char *_log_file;
1308  _log_file = str_fmt("%sopenttd.log", _personal_dir);
1309 #else /* ENABLE_NETWORK */
1310  /* If we don't have networking, we don't need to make the directory. But
1311  * if it exists we keep it, otherwise remove it from the search paths. */
1312  if (!FileExists(_searchpaths[SP_AUTODOWNLOAD_DIR])) {
1313  free(_searchpaths[SP_AUTODOWNLOAD_DIR]);
1314  _searchpaths[SP_AUTODOWNLOAD_DIR] = NULL;
1315  }
1316 #endif /* ENABLE_NETWORK */
1317 }
1318 
1323 void SanitizeFilename(char *filename)
1324 {
1325  for (; *filename != '\0'; filename++) {
1326  switch (*filename) {
1327  /* The following characters are not allowed in filenames
1328  * on at least one of the supported operating systems: */
1329  case ':': case '\\': case '*': case '?': case '/':
1330  case '<': case '>': case '|': case '"':
1331  *filename = '_';
1332  break;
1333  }
1334  }
1335 }
1336 
1345 void *ReadFileToMem(const char *filename, size_t *lenp, size_t maxsize)
1346 {
1347  FILE *in = fopen(filename, "rb");
1348  if (in == NULL) return NULL;
1349 
1350  fseek(in, 0, SEEK_END);
1351  size_t len = ftell(in);
1352  fseek(in, 0, SEEK_SET);
1353  if (len > maxsize) {
1354  fclose(in);
1355  return NULL;
1356  }
1357  byte *mem = MallocT<byte>(len + 1);
1358  mem[len] = 0;
1359  if (fread(mem, len, 1, in) != 1) {
1360  fclose(in);
1361  free(mem);
1362  return NULL;
1363  }
1364  fclose(in);
1365 
1366  *lenp = len;
1367  return mem;
1368 }
1369 
1376 static bool MatchesExtension(const char *extension, const char *filename)
1377 {
1378  if (extension == NULL) return true;
1379 
1380  const char *ext = strrchr(filename, extension[0]);
1381  return ext != NULL && strcasecmp(ext, extension) == 0;
1382 }
1383 
1393 static uint ScanPath(FileScanner *fs, const char *extension, const char *path, size_t basepath_length, bool recursive)
1394 {
1395  extern bool FiosIsValidFile(const char *path, const struct dirent *ent, struct stat *sb);
1396 
1397  uint num = 0;
1398  struct stat sb;
1399  struct dirent *dirent;
1400  DIR *dir;
1401 
1402  if (path == NULL || (dir = ttd_opendir(path)) == NULL) return 0;
1403 
1404  while ((dirent = readdir(dir)) != NULL) {
1405  const char *d_name = FS2OTTD(dirent->d_name);
1406  char filename[MAX_PATH];
1407 
1408  if (!FiosIsValidFile(path, dirent, &sb)) continue;
1409 
1410  seprintf(filename, lastof(filename), "%s%s", path, d_name);
1411 
1412  if (S_ISDIR(sb.st_mode)) {
1413  /* Directory */
1414  if (!recursive) continue;
1415  if (strcmp(d_name, ".") == 0 || strcmp(d_name, "..") == 0) continue;
1416  if (!AppendPathSeparator(filename, lastof(filename))) continue;
1417  num += ScanPath(fs, extension, filename, basepath_length, recursive);
1418  } else if (S_ISREG(sb.st_mode)) {
1419  /* File */
1420  if (MatchesExtension(extension, filename) && fs->AddFile(filename, basepath_length, NULL)) num++;
1421  }
1422  }
1423 
1424  closedir(dir);
1425 
1426  return num;
1427 }
1428 
1435 static uint ScanTar(FileScanner *fs, const char *extension, TarFileList::iterator tar)
1436 {
1437  uint num = 0;
1438  const char *filename = (*tar).first.c_str();
1439 
1440  if (MatchesExtension(extension, filename) && fs->AddFile(filename, 0, (*tar).second.tar_filename)) num++;
1441 
1442  return num;
1443 }
1444 
1454 uint FileScanner::Scan(const char *extension, Subdirectory sd, bool tars, bool recursive)
1455 {
1456  this->subdir = sd;
1457 
1458  Searchpath sp;
1459  char path[MAX_PATH];
1460  TarFileList::iterator tar;
1461  uint num = 0;
1462 
1463  FOR_ALL_SEARCHPATHS(sp) {
1464  /* Don't search in the working directory */
1465  if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
1466 
1467  FioAppendDirectory(path, lastof(path), sp, sd);
1468  num += ScanPath(this, extension, path, strlen(path), recursive);
1469  }
1470 
1471  if (tars && sd != NO_DIRECTORY) {
1472  FOR_ALL_TARS(tar, sd) {
1473  num += ScanTar(this, extension, tar);
1474  }
1475  }
1476 
1477  switch (sd) {
1478  case BASESET_DIR:
1479  num += this->Scan(extension, OLD_GM_DIR, tars, recursive);
1480  /* FALL THROUGH */
1481  case NEWGRF_DIR:
1482  num += this->Scan(extension, OLD_DATA_DIR, tars, recursive);
1483  break;
1484 
1485  default: break;
1486  }
1487 
1488  return num;
1489 }
1490 
1499 uint FileScanner::Scan(const char *extension, const char *directory, bool recursive)
1500 {
1501  char path[MAX_PATH];
1502  strecpy(path, directory, lastof(path));
1503  if (!AppendPathSeparator(path, lastof(path))) return 0;
1504  return ScanPath(this, extension, path, strlen(path), recursive);
1505 }