You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

716 lines
20KB

  1. /*
  2. XCSV - X Character Separated Values (.???)
  3. A hopefully not too feeble attempt at parsing whatever separated values
  4. files into the waypoint structure and back out again. This is a config-
  5. file wrapper around csv_util.c.
  6. Copyright (C) 2002 Alex Mottram (geo_alexm at cox-internet.com)
  7. This program is free software; you can redistribute it and/or modify
  8. it under the terms of the GNU General Public License as published by
  9. the Free Software Foundation; either version 2 of the License, or
  10. (at your option) any later version.
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. GNU General Public License for more details.
  15. You should have received a copy of the GNU General Public License
  16. along with this program; if not, write to the Free Software
  17. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111 USA
  18. */
  19. #include <QtCore/QTextCodec>
  20. #include <QtCore/QTextStream>
  21. #include "defs.h"
  22. #include "csv_util.h"
  23. #include "jeeps/gpsmath.h"
  24. #include "src/core/file.h"
  25. #include "src/core/logging.h"
  26. #include <ctype.h>
  27. #include <stdlib.h>
  28. #if CSVFMTS_ENABLED
  29. #define MYNAME "XCSV"
  30. #define ISSTOKEN(a,b) (strncmp(a,b, strlen(b)) == 0)
  31. static char* styleopt = NULL;
  32. static char* snlenopt = NULL;
  33. static char* snwhiteopt = NULL;
  34. static char* snupperopt = NULL;
  35. static char* snuniqueopt = NULL;
  36. char* prefer_shortnames = NULL;
  37. char* xcsv_urlbase = NULL;
  38. static char* opt_datum;
  39. static const char* intstylebuf = NULL;
  40. static
  41. arglist_t xcsv_args[] = {
  42. {
  43. "style", &styleopt, "Full path to XCSV style file", NULL,
  44. ARGTYPE_FILE | ARGTYPE_REQUIRED, ARG_NOMINMAX
  45. },
  46. {
  47. "snlen", &snlenopt, "Max synthesized shortname length", NULL,
  48. ARGTYPE_INT, "1", NULL
  49. },
  50. {
  51. "snwhite", &snwhiteopt, "Allow whitespace synth. shortnames",
  52. NULL, ARGTYPE_BOOL, ARG_NOMINMAX
  53. },
  54. {
  55. "snupper", &snupperopt, "UPPERCASE synth. shortnames",
  56. NULL, ARGTYPE_BOOL, ARG_NOMINMAX
  57. },
  58. {
  59. "snunique", &snuniqueopt, "Make synth. shortnames unique",
  60. NULL, ARGTYPE_BOOL, ARG_NOMINMAX
  61. },
  62. {
  63. "urlbase", &xcsv_urlbase, "Basename prepended to URL on output",
  64. NULL, ARGTYPE_STRING, ARG_NOMINMAX
  65. },
  66. {
  67. "prefer_shortnames", &prefer_shortnames,
  68. "Use shortname instead of description",
  69. NULL, ARGTYPE_BOOL, ARG_NOMINMAX
  70. },
  71. {
  72. "datum", &opt_datum, "GPS datum (def. WGS 84)",
  73. "WGS 84", ARGTYPE_STRING, ARG_NOMINMAX
  74. },
  75. ARG_TERMINATOR
  76. };
  77. /* a table of config file constants mapped to chars */
  78. static
  79. char_map_t xcsv_char_table[] = {
  80. { "COMMA", "," },
  81. { "COMMASPACE", ", " },
  82. { "SINGLEQUOTE", "'" },
  83. { "DOUBLEQUOTE", "\"" },
  84. { "COLON", ":" },
  85. { "SEMICOLON", ";" },
  86. { "NEWLINE", "\n" },
  87. { "CR", "\n" },
  88. { "CRNEWLINE", "\r\n" },
  89. { "TAB", "\t" },
  90. { "SPACE", " " },
  91. { "HASH", "#" },
  92. { "WHITESPACE", "\\w" },
  93. { "PIPE", "|" },
  94. { NULL, NULL }
  95. };
  96. void
  97. xcsv_destroy_style(void)
  98. {
  99. queue* elem, *tmp;
  100. field_map_t* fmp;
  101. int internal = 0;
  102. /*
  103. * If this xcsv_file struct came from a file we can free it all.
  104. * If not, we can at least free the queue elements.
  105. */
  106. /* destroy the prologue */
  107. xcsv_file.epilogue.clear();
  108. /* destroy the epilogue */
  109. xcsv_file.epilogue.clear();
  110. /* destroy the ifields */
  111. QUEUE_FOR_EACH(&xcsv_file.ifield, elem, tmp) {
  112. fmp = (field_map_t*) elem;
  113. if (fmp->key) {
  114. xfree(fmp->key);
  115. }
  116. if (fmp->val) {
  117. xfree(fmp->val);
  118. }
  119. if (fmp->printfc) {
  120. xfree(fmp->printfc);
  121. }
  122. if (elem) {
  123. xfree(elem);
  124. }
  125. }
  126. /* destroy the ofields, if they are not re-mapped to ifields. */
  127. if (xcsv_file.ofield != &xcsv_file.ifield) {
  128. QUEUE_FOR_EACH(xcsv_file.ofield, elem, tmp) {
  129. fmp = (field_map_t*) elem;
  130. if (fmp->key) {
  131. xfree(fmp->key);
  132. }
  133. if (fmp->val) {
  134. xfree(fmp->val);
  135. }
  136. if (fmp->printfc) {
  137. xfree(fmp->printfc);
  138. }
  139. if (elem) {
  140. xfree(elem);
  141. }
  142. }
  143. if (xcsv_file.ofield) {
  144. xfree(xcsv_file.ofield);
  145. }
  146. }
  147. /* other alloc'd glory */
  148. xcsv_file.field_delimiter = QString();
  149. xcsv_file.field_encloser = QString();
  150. xcsv_file.record_delimiter = QString();
  151. xcsv_file.badchars = QString();
  152. if (xcsv_file.description) {
  153. xfree(xcsv_file.description);
  154. }
  155. if (xcsv_file.extension) {
  156. xfree(xcsv_file.extension);
  157. }
  158. if (xcsv_file.mkshort_handle) {
  159. mkshort_del_handle(&xcsv_file.mkshort_handle);
  160. }
  161. /* return everything to zeros */
  162. internal = xcsv_file.is_internal;
  163. xcsv_file.is_internal = internal;
  164. }
  165. const char*
  166. xcsv_get_char_from_constant_table(char* key)
  167. {
  168. char_map_t* cm = xcsv_char_table;
  169. while ((cm->key) && (strcmp(key, cm->key) != 0)) {
  170. cm++;
  171. }
  172. return (cm->chars);
  173. }
  174. static void
  175. xcsv_parse_style_line(char* sbuff)
  176. {
  177. int i, linecount = 0;
  178. char* s, *sp;
  179. char* p;
  180. const char* cp;
  181. char* key, *val, *pfc;
  182. /*
  183. * tokens should be parsed longest to shortest, unless something
  184. * requires a previously set value. This way something like
  185. * SHORT and SHORTNAME don't collide.
  186. */
  187. /* whack off any comments */
  188. if ((p = strchr(sbuff, '#')) != NULL) {
  189. if ((p > sbuff) && p[-1] == '\\') {
  190. memmove(p-1, p, strlen(p));
  191. p[strlen(p)-1] = '\0';
  192. } else {
  193. *p = '\0';
  194. }
  195. }
  196. if (strlen(sbuff)) {
  197. if (ISSTOKEN(sbuff, "FIELD_DELIMITER")) {
  198. sp = csv_stringtrim(&sbuff[16], "\"", 1);
  199. cp = xcsv_get_char_from_constant_table(sp);
  200. if (cp) {
  201. xcsv_file.field_delimiter = cp;
  202. xfree(sp);
  203. } else {
  204. xcsv_file.field_delimiter = sp;
  205. }
  206. p = csv_stringtrim(CSTR(xcsv_file.field_delimiter), " ", 0);
  207. /* field delimiters are always bad characters */
  208. if (0 == strcmp(p, "\\w")) {
  209. xcsv_file.badchars = " \n\r";
  210. } else {
  211. xcsv_file.badchars += p;
  212. }
  213. xfree(p);
  214. } else
  215. if (ISSTOKEN(sbuff, "FIELD_ENCLOSER")) {
  216. sp = csv_stringtrim(&sbuff[15], "\"", 1);
  217. cp = xcsv_get_char_from_constant_table(sp);
  218. if (cp) {
  219. xcsv_file.field_encloser = cp;
  220. xfree(sp);
  221. } else {
  222. xcsv_file.field_encloser = sp;
  223. }
  224. p = csv_stringtrim(CSTR(xcsv_file.field_encloser), " ", 0);
  225. xcsv_file.badchars += p;
  226. xfree(p);
  227. } else
  228. if (ISSTOKEN(sbuff, "RECORD_DELIMITER")) {
  229. sp = csv_stringtrim(&sbuff[17], "\"", 1);
  230. cp = xcsv_get_char_from_constant_table(sp);
  231. if (cp) {
  232. xcsv_file.record_delimiter = cp;
  233. xfree(sp);
  234. } else {
  235. xcsv_file.record_delimiter = sp;
  236. }
  237. /* record delimiters are always bad characters */
  238. p = csv_stringtrim(CSTR(xcsv_file.record_delimiter), " ", 0);
  239. xcsv_file.badchars += p;
  240. xfree(p);
  241. } else
  242. if (ISSTOKEN(sbuff, "FORMAT_TYPE")) {
  243. const char* p;
  244. for (p = &sbuff[11]; *p && isspace(*p); p++) {
  245. ;
  246. }
  247. if (ISSTOKEN(p, "INTERNAL")) {
  248. xcsv_file.type = ff_type_internal;
  249. }
  250. /* this is almost inconcievable... */
  251. if (ISSTOKEN(p, "SERIAL")) {
  252. xcsv_file.type = ff_type_serial;
  253. }
  254. } else
  255. if (ISSTOKEN(sbuff, "DESCRIPTION")) {
  256. xcsv_file.description = csv_stringtrim(&sbuff[11],"", 0);
  257. } else
  258. if (ISSTOKEN(sbuff, "EXTENSION")) {
  259. xcsv_file.extension = csv_stringtrim(&sbuff[10],"", 0);
  260. } else
  261. if (ISSTOKEN(sbuff, "SHORTLEN")) {
  262. if (xcsv_file.mkshort_handle) {
  263. setshort_length(xcsv_file.mkshort_handle, atoi(&sbuff[9]));
  264. }
  265. } else
  266. if (ISSTOKEN(sbuff, "SHORTWHITE")) {
  267. if (xcsv_file.mkshort_handle) {
  268. setshort_whitespace_ok(xcsv_file.mkshort_handle, atoi(&sbuff[12]));
  269. }
  270. } else
  271. if (ISSTOKEN(sbuff, "BADCHARS")) {
  272. sp = csv_stringtrim(&sbuff[9], "\"", 1);
  273. cp = xcsv_get_char_from_constant_table(sp);
  274. if (cp) {
  275. p = xstrdup(cp);
  276. xfree(sp);
  277. } else {
  278. p = sp;
  279. }
  280. xcsv_file.badchars += p;
  281. xfree(p);
  282. } else
  283. if (ISSTOKEN(sbuff, "PROLOGUE")) {
  284. xcsv_prologue_add(sbuff + 9);
  285. } else
  286. if (ISSTOKEN(sbuff, "EPILOGUE")) {
  287. xcsv_epilogue_add(sbuff + 9);
  288. } else
  289. if (ISSTOKEN(sbuff, "ENCODING")) {
  290. p = csv_stringtrim(&sbuff[8], "\"", 1);
  291. xcsv_file.codec = QTextCodec::codecForName(p);
  292. if (!xcsv_file.codec) {
  293. Fatal() << "Unsupported character set '" << p << "'.";
  294. }
  295. xfree(p);
  296. } else
  297. if (ISSTOKEN(sbuff, "DATUM")) {
  298. p = csv_stringtrim(&sbuff[5], "\"", 1);
  299. xcsv_file.gps_datum = GPS_Lookup_Datum_Index(p);
  300. is_fatal(xcsv_file.gps_datum < 0, MYNAME ": datum \"%s\" is not supported.", p);
  301. xfree(p);
  302. } else
  303. if (ISSTOKEN(sbuff, "DATATYPE")) {
  304. p = csv_stringtrim(&sbuff[8], "\"", 1);
  305. if (case_ignore_strcmp(p, "TRACK") == 0) {
  306. xcsv_file.datatype = trkdata;
  307. } else if (case_ignore_strcmp(p, "ROUTE") == 0) {
  308. xcsv_file.datatype = rtedata;
  309. } else if (case_ignore_strcmp(p, "WAYPOINT") == 0) {
  310. xcsv_file.datatype = wptdata;
  311. } else {
  312. fatal(MYNAME ": Unknown data type \"%s\"!\n", p);
  313. }
  314. xfree(p);
  315. } else
  316. if (ISSTOKEN(sbuff, "IFIELD")) {
  317. key = val = pfc = NULL;
  318. s = csv_lineparse(&sbuff[6], ",", "", linecount);
  319. i = 0;
  320. while (s) {
  321. switch (i) {
  322. case 0:
  323. /* key */
  324. key = csv_stringtrim(s, "\"", 1);
  325. break;
  326. case 1:
  327. /* default value */
  328. val = csv_stringtrim(s, "\"", 1);
  329. break;
  330. case 2:
  331. /* printf conversion */
  332. pfc = csv_stringtrim(s, "\"", 1);
  333. break;
  334. default:
  335. break;
  336. }
  337. i++;
  338. s = csv_lineparse(NULL, ",", "", linecount);
  339. }
  340. xcsv_ifield_add(key, val, pfc);
  341. } else
  342. /*
  343. * as OFIELDs are implemented as an after-thought, I'll
  344. * leave this as it's own parsing for now. We could
  345. * change the world on ifield vs ofield format later..
  346. */
  347. if (ISSTOKEN(sbuff, "OFIELD")) {
  348. int options = 0;
  349. key = val = pfc = NULL;
  350. s = csv_lineparse(&sbuff[6], ",", "", linecount);
  351. i = 0;
  352. while (s) {
  353. switch (i) {
  354. case 0:
  355. /* key */
  356. key = csv_stringtrim(s, "\"", 1);
  357. break;
  358. case 1:
  359. /* default value */
  360. val = csv_stringtrim(s, "\"", 1);
  361. break;
  362. case 2:
  363. /* printf conversion */
  364. pfc = csv_stringtrim(s, "\"", 1);
  365. break;
  366. case 3:
  367. /* Any additional options. */
  368. if (strstr(s, "no_delim_before")) {
  369. options |= OPTIONS_NODELIM;
  370. }
  371. if (strstr(s, "absolute")) {
  372. options |= OPTIONS_ABSOLUTE;
  373. }
  374. if (strstr(s, "optional")) {
  375. options |= OPTIONS_OPTIONAL;
  376. }
  377. default:
  378. break;
  379. }
  380. i++;
  381. s = csv_lineparse(NULL, ",", "", linecount);
  382. }
  383. xcsv_ofield_add(key, val, pfc, options);
  384. }
  385. }
  386. }
  387. /*
  388. * A wrapper for xcsv_parse_style_line that reads until it hits
  389. * a terminating null. Makes multiple calls to that function so
  390. * that "ignore to end of line" comments work right.
  391. */
  392. static void
  393. xcsv_parse_style_buff(const char* sbuff)
  394. {
  395. // FIXME: should not be a static buf. Should not be a raw character
  396. // buffer at all!
  397. char ibuf[4096];
  398. char* ibufp;
  399. size_t i;
  400. while (*sbuff) {
  401. ibuf[0] = 0;
  402. i = 0;
  403. for (ibufp = ibuf; *sbuff != '\n' && i++ < sizeof(ibuf);) {
  404. *ibufp++ = *sbuff++;
  405. }
  406. while (*sbuff == '\n' || *sbuff == '\r') {
  407. sbuff++;
  408. }
  409. *ibufp = 0;
  410. xcsv_parse_style_line(ibuf);
  411. }
  412. }
  413. static void
  414. xcsv_read_style(const char* fname)
  415. {
  416. char* sbuff;
  417. gbfile* fp;
  418. xcsv_file_init();
  419. fp = gbfopen(fname, "rb", MYNAME);
  420. while ((sbuff = gbfgetstr(fp))) {
  421. sbuff = lrtrim(sbuff);
  422. xcsv_parse_style_line(sbuff);
  423. }
  424. while (!gbfeof(fp));
  425. /* if we have no output fields, use input fields as output fields */
  426. if (xcsv_file.ofield_ct == 0) {
  427. if (xcsv_file.ofield) {
  428. xfree(xcsv_file.ofield);
  429. }
  430. xcsv_file.ofield = &xcsv_file.ifield;
  431. xcsv_file.ofield_ct = xcsv_file.ifield_ct;
  432. }
  433. gbfclose(fp);
  434. }
  435. /*
  436. * Passed a pointer to an internal buffer that would be identical
  437. * to the series of bytes that would be in a style file, we set up
  438. * the xcsv parser and make it ready for general use.
  439. */
  440. void
  441. xcsv_read_internal_style(const char* style_buf)
  442. {
  443. xcsv_file_init();
  444. xcsv_file.is_internal = 1;
  445. xcsv_parse_style_buff(style_buf);
  446. /* if we have no output fields, use input fields as output fields */
  447. if (xcsv_file.ofield_ct == 0) {
  448. if (xcsv_file.ofield) {
  449. xfree(xcsv_file.ofield);
  450. }
  451. xcsv_file.ofield = &xcsv_file.ifield;
  452. xcsv_file.ofield_ct = xcsv_file.ifield_ct;
  453. }
  454. }
  455. void
  456. xcsv_setup_internal_style(const char* style_buf)
  457. {
  458. xcsv_file_init();
  459. xcsv_destroy_style();
  460. xcsv_file.is_internal = !!style_buf;
  461. intstylebuf = style_buf;
  462. }
  463. static void
  464. xcsv_rd_init(const QString& fname)
  465. {
  466. /*
  467. * if we don't have an internal style defined, we need to
  468. * read it from a user-supplied style file, or die trying.
  469. */
  470. if (xcsv_file.is_internal) {
  471. xcsv_read_internal_style(intstylebuf);
  472. } else {
  473. if (!styleopt) {
  474. fatal(MYNAME ": XCSV input style not declared. Use ... -i xcsv,style=path/to/file.style\n");
  475. }
  476. xcsv_read_style(styleopt);
  477. }
  478. if ((xcsv_file.datatype == 0) || (xcsv_file.datatype == wptdata)) {
  479. if (global_opts.masked_objective & (TRKDATAMASK|RTEDATAMASK)) {
  480. warning(MYNAME " attempt to read %s as a track or route, but this format only supports waypoints on read. Reading as waypoints instead.\n", qPrintable(fname));
  481. }
  482. }
  483. xcsv_file.file = new gpsbabel::File(fname);
  484. xcsv_file.file->open(QFile::ReadOnly);
  485. xcsv_file.stream = new QTextStream(xcsv_file.file);
  486. if (xcsv_file.codec) {
  487. xcsv_file.stream->setCodec(xcsv_file.codec);
  488. } else {
  489. // default to UTF-8.
  490. xcsv_file.stream->setCodec("UTF-8");
  491. xcsv_file.stream->setAutoDetectUnicode(true);
  492. }
  493. xcsv_file.gps_datum = GPS_Lookup_Datum_Index(opt_datum);
  494. is_fatal(xcsv_file.gps_datum < 0, MYNAME ": datum \"%s\" is not supported.", opt_datum);
  495. }
  496. static void
  497. xcsv_rd_deinit(void)
  498. {
  499. xcsv_file.file->close();
  500. delete xcsv_file.file;
  501. xcsv_file.file = NULL;
  502. delete xcsv_file.stream;
  503. xcsv_file.stream = NULL;
  504. xcsv_file.codec = NULL;
  505. xcsv_destroy_style();
  506. }
  507. static void
  508. xcsv_wr_init(const QString& fname)
  509. {
  510. /* if we don't have an internal style defined, we need to
  511. * read it from a user-supplied style file, or die trying.
  512. * 8/19 - add test for styleopt to ensure that a write of a style
  513. * after a read of a style works.
  514. */
  515. if (xcsv_file.is_internal && !styleopt) {
  516. xcsv_read_internal_style(intstylebuf);
  517. } else {
  518. if (!styleopt) {
  519. fatal(MYNAME ": XCSV output style not declared. Use ... -o xcsv,style=path/to/file.style\n");
  520. }
  521. xcsv_read_style(styleopt);
  522. }
  523. xcsv_file.file = new gpsbabel::File(fname);
  524. xcsv_file.file->open(QFile::WriteOnly | QFile::Text);
  525. xcsv_file.stream = new QTextStream(xcsv_file.file);
  526. if (xcsv_file.codec) {
  527. xcsv_file.stream->setCodec(xcsv_file.codec);
  528. // enable bom for all UTF codecs except UTF-8
  529. if (xcsv_file.codec->mibEnum() != 106) {
  530. xcsv_file.stream->setGenerateByteOrderMark(true);
  531. }
  532. } else {
  533. // emulate gbfputs which assumes UTF-8.
  534. xcsv_file.stream->setCodec("UTF-8");
  535. }
  536. xcsv_file.fname = fname;
  537. /* set mkshort options from the command line */
  538. if (global_opts.synthesize_shortnames) {
  539. if (snlenopt) {
  540. setshort_length(xcsv_file.mkshort_handle, atoi(snlenopt));
  541. }
  542. if (snwhiteopt) {
  543. setshort_whitespace_ok(xcsv_file.mkshort_handle, atoi(snwhiteopt));
  544. }
  545. if (snupperopt) {
  546. setshort_mustupper(xcsv_file.mkshort_handle, atoi(snupperopt));
  547. }
  548. if (snuniqueopt) {
  549. setshort_mustuniq(xcsv_file.mkshort_handle, atoi(snuniqueopt));
  550. }
  551. setshort_badchars(xcsv_file.mkshort_handle, CSTR(xcsv_file.badchars));
  552. }
  553. xcsv_file.gps_datum = GPS_Lookup_Datum_Index(opt_datum);
  554. is_fatal(xcsv_file.gps_datum < 0, MYNAME ": datum \"%s\" is not supported.", opt_datum);
  555. }
  556. static void
  557. xcsv_wr_position_init(const QString& fname)
  558. {
  559. xcsv_wr_init(fname);
  560. }
  561. static void
  562. xcsv_wr_deinit(void)
  563. {
  564. xcsv_file.stream->flush();
  565. xcsv_file.file->close();
  566. delete xcsv_file.file;
  567. xcsv_file.file = NULL;
  568. delete xcsv_file.stream;
  569. xcsv_file.stream = NULL;
  570. xcsv_file.codec = NULL;
  571. xcsv_destroy_style();
  572. }
  573. static void
  574. xcsv_wr_position_deinit(void)
  575. {
  576. xcsv_wr_deinit();
  577. }
  578. static void
  579. xcsv_wr_position(Waypoint* wpt)
  580. {
  581. /* Tweak incoming name if we don't have a fix */
  582. switch (wpt->fix) {
  583. case fix_none:
  584. wpt->shortname = "ESTIMATED Position";
  585. break;
  586. default:
  587. break;
  588. }
  589. waypt_add(wpt);
  590. xcsv_data_write();
  591. waypt_del(wpt);
  592. xcsv_file.stream->flush();
  593. }
  594. ff_vecs_t xcsv_vecs = {
  595. ff_type_internal,
  596. FF_CAP_RW_WPT, /* This is a bit of a lie for now... */
  597. xcsv_rd_init,
  598. xcsv_wr_init,
  599. xcsv_rd_deinit,
  600. xcsv_wr_deinit,
  601. xcsv_data_read,
  602. xcsv_data_write,
  603. NULL,
  604. xcsv_args,
  605. CET_CHARSET_ASCII, 0, /* CET-REVIEW */
  606. { NULL, NULL, NULL, xcsv_wr_position_init, xcsv_wr_position, xcsv_wr_position_deinit }
  607. };
  608. #else
  609. void xcsv_read_internal_style(const char* style_buf) {}
  610. void xcsv_setup_internal_style(const char* style_buf) {}
  611. #endif //CSVFMTS_ENABLED