parsers.rs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. use anyhow::Result;
  2. use regex::{Regex, RegexSet, Match, Captures};
  3. /// Single ISBN parsed from a string
  4. #[derive(Debug, PartialEq)]
  5. pub struct ISBN {
  6. pub text: String,
  7. pub tags: Vec<String>
  8. }
  9. /// Result of parsing an ISBN string
  10. #[derive(Debug, PartialEq)]
  11. pub enum ParseResult {
  12. Valid(Vec<ISBN>, String),
  13. Ignored(String),
  14. Unmatched(String)
  15. }
  16. /// Regular expressions for unparsable ISBN strings to ignore.
  17. /// This cleans up warning displays.
  18. static IGNORES: &'static [&'static str] = &[
  19. r"^[$]?[[:digit:]., ]+(?:[a-zA-Z*]{1,4})?(\s+\(.*?\))?$",
  20. r"^[[:digit:].]+(/[[:digit:].]+)+$",
  21. r"^[A-Z]-[A-Z]-\d{8,}",
  22. r"^\s*$"
  23. ];
  24. /// Definitions for parsing ISBN strings.
  25. pub struct ParserDefs {
  26. /// Matcher for text that may appear before an ISBN
  27. lead: Regex,
  28. /// Matcher for a single ISBN
  29. isbn: Regex,
  30. /// Matcher for a "tag" after an ISBN
  31. tag: Regex,
  32. /// Matcher for separarators between multiple tags
  33. tag_sep: Regex,
  34. /// Matcher for text to skip before possibly reading another ISBN
  35. tail_skip: Regex,
  36. /// Matcher for characters to remove from a parsed ISBN
  37. clean: Regex,
  38. /// Matcher for text that is known not to contain any parseable ISBNs
  39. unmatch_ignore: RegexSet
  40. }
  41. impl ParserDefs {
  42. pub fn new() -> ParserDefs {
  43. fn cre(p: &str) -> Regex {
  44. // we use unwrap instead of result since regex compile failure is a programming error
  45. Regex::new(p).unwrap()
  46. }
  47. ParserDefs {
  48. lead: cre(r"^[;.]?\s*(?:[a-z]\s+|\(\d+\)\s+|\*|ISBN\s+)?"),
  49. isbn: cre(r"^([\p{Nonspacing Mark}0-9-]{8,}[Xx]?|[0-9]{1,5}(?:[a-zA-Z]+|[ +])[0-9-]{4,})"),
  50. tag: cre(r"^\s*[(\[](.+?)[)\]]"),
  51. tag_sep: cre(r"\s*:\s*"),
  52. tail_skip: cre(r"^\s*[;:/.]?"),
  53. clean: cre(r"[\p{Nonspacing Mark}a-wyzA-WYZ -]"),
  54. unmatch_ignore: RegexSet::new(IGNORES).unwrap()
  55. }
  56. }
  57. /// Create a new parser to parse a string.
  58. pub fn create_parser<'p, 's>(&'p self, s: &'s str) -> IsbnParser<'p, 's> {
  59. IsbnParser {
  60. defs: self,
  61. string: s,
  62. position: 0
  63. }
  64. }
  65. /// Parse a string
  66. pub fn parse(&self, s: &str) -> ParseResult {
  67. let mut parser = self.create_parser(s);
  68. parser.read_all()
  69. }
  70. }
  71. pub struct IsbnParser<'p, 's> {
  72. defs: &'p ParserDefs,
  73. string: &'s str,
  74. position: usize
  75. }
  76. fn preclean(s: &str) -> String {
  77. let mut res = String::with_capacity(s.len());
  78. for c in s.chars() {
  79. if c.is_ascii() {
  80. res.push(c);
  81. }
  82. }
  83. res
  84. }
  85. impl <'p, 's> IsbnParser<'p, 's> {
  86. /// Get the remaining (unparsed) text from the parser
  87. fn remaining(&self) -> &'s str {
  88. &self.string[self.position..]
  89. }
  90. /// Advance the parse position to the end of a regex patch, if possible.
  91. fn eat(&mut self, rex: &Regex) -> bool {
  92. let slice = self.remaining();
  93. if let Some(m) = rex.find(slice) {
  94. self.advance(m.end());
  95. true
  96. } else {
  97. false
  98. }
  99. }
  100. /// See if a regex matches.
  101. fn peek(&self, rex: &Regex) -> Option<Match> {
  102. let slice = self.remaining();
  103. rex.find(slice)
  104. }
  105. /// See if a regex matches, and advance if it does.
  106. fn read(&mut self, rex: &Regex) -> Option<Match<'s>> {
  107. let slice = self.remaining();
  108. let res = rex.find(slice);
  109. if let Some(m) = res {
  110. self.advance(m.end());
  111. }
  112. res
  113. }
  114. /// Read with capture groups
  115. fn read_cap(&mut self, rex: &Regex) -> Option<Captures<'s>> {
  116. let slice = self.remaining();
  117. let res = rex.captures(slice);
  118. if let Some(ref m) = res {
  119. self.advance(m.get(0).unwrap().end());
  120. }
  121. res
  122. }
  123. /// Advance the parse position by `n` characters.
  124. fn advance(&mut self, n: usize) {
  125. self.position += n;
  126. }
  127. fn is_empty(&self) -> bool {
  128. self.position == self.string.len()
  129. }
  130. /// Read a single ISBN
  131. fn read_isbn(&mut self) -> Option<ISBN> {
  132. self.eat(&self.defs.lead);
  133. self.read(&self.defs.isbn).map(|m| ISBN {
  134. text: self.defs.clean.replace_all(m.as_str(), "").to_string(),
  135. tags: self.read_tags()
  136. })
  137. }
  138. /// Read tags (assuming an ISBN has just been read)
  139. fn read_tags(&mut self) -> Vec<String> {
  140. let mut tags = Vec::new();
  141. while let Some(m) = self.read_cap(&self.defs.tag) {
  142. let tag = m.get(1).unwrap().as_str();
  143. for t in self.defs.tag_sep.split(tag) {
  144. tags.push(t.to_owned());
  145. }
  146. }
  147. tags
  148. }
  149. /// Read all ISBNs
  150. fn read_all(&mut self) -> ParseResult {
  151. let mut isbns = Vec::new();
  152. while let Some(res) = self.read_isbn() {
  153. isbns.push(res);
  154. // advance through our skip
  155. self.eat(&self.defs.tail_skip);
  156. }
  157. if isbns.is_empty() {
  158. if self.defs.unmatch_ignore.is_match(self.string) {
  159. ParseResult::Ignored(self.string.to_owned())
  160. } else {
  161. ParseResult::Unmatched(self.string.to_owned())
  162. }
  163. } else {
  164. ParseResult::Valid(isbns, self.remaining().to_owned())
  165. }
  166. }
  167. }
  168. #[test]
  169. fn test_preclean_keep() {
  170. assert_eq!(preclean("foo").as_str(), "foo");
  171. }
  172. #[test]
  173. fn test_preclean_caron() {
  174. let src = "349̌224010X";
  175. let isbn = "349224010X";
  176. assert_eq!(preclean(src).as_str(), isbn);
  177. }
  178. #[test]
  179. fn test_parser_initial() {
  180. let defs = ParserDefs::new();
  181. let target = "jimbob";
  182. let parser = defs.create_parser(target);
  183. assert_eq!(parser.position, 0);
  184. assert_eq!(parser.string, target);
  185. assert_eq!(parser.remaining(), target);
  186. }
  187. #[test]
  188. fn test_eat_nomatch() {
  189. let defs = ParserDefs::new();
  190. let target = "jimbob";
  191. let pat = Regex::new(r"^\d").unwrap();
  192. let mut parser = defs.create_parser(target);
  193. assert!(!parser.eat(&pat));
  194. assert_eq!(parser.position, 0);
  195. }
  196. #[test]
  197. fn test_eat_match() {
  198. let defs = ParserDefs::new();
  199. let target = "jimbob";
  200. let pat = Regex::new(r"^jim").unwrap();
  201. let mut parser = defs.create_parser(target);
  202. assert!(parser.eat(&pat));
  203. assert_eq!(parser.position, 3);
  204. assert!(!parser.is_empty());
  205. assert_eq!(parser.remaining(), "bob");
  206. }
  207. #[test]
  208. fn test_eat_later() {
  209. let defs = ParserDefs::new();
  210. let target = "jimjim";
  211. let pat = Regex::new(r"^jim").unwrap();
  212. let mut parser = defs.create_parser(target);
  213. assert!(parser.eat(&pat));
  214. assert_eq!(parser.position, 3);
  215. assert!(parser.eat(&pat));
  216. assert_eq!(parser.position, 6);
  217. assert!(parser.is_empty());
  218. // eating again fails
  219. assert!(!parser.eat(&pat));
  220. assert_eq!(parser.remaining(), "");
  221. }
  222. #[test]
  223. fn test_scan_empty() {
  224. let defs = ParserDefs::new();
  225. let mut parser = defs.create_parser("");
  226. assert_eq!(parser.read_isbn(), None);
  227. }
  228. #[test]
  229. fn test_parse_empty() {
  230. let defs = ParserDefs::new();
  231. let res = defs.parse("");
  232. assert_eq!(res, ParseResult::Ignored("".to_owned()));
  233. }
  234. #[test]
  235. fn test_scan_ws() {
  236. let defs = ParserDefs::new();
  237. let mut parser = defs.create_parser(" ");
  238. assert_eq!(parser.read_isbn(), None);
  239. }
  240. #[test]
  241. fn test_parse_ws() {
  242. let defs = ParserDefs::new();
  243. let res = defs.parse(" ");
  244. assert_eq!(res, ParseResult::Ignored(" ".to_owned()));
  245. }
  246. #[test]
  247. fn test_parse_isbn() {
  248. let isbn = "349224010X";
  249. let defs = ParserDefs::new();
  250. let mut parser = defs.create_parser(isbn);
  251. let scan = parser.read_isbn();
  252. assert!(scan.is_some());
  253. let scan = scan.unwrap();
  254. assert_eq!(scan.text, isbn);
  255. assert_eq!(scan.tags.len(), 0);
  256. assert_eq!(parser.position, isbn.len());
  257. assert!(parser.is_empty());
  258. let res = defs.parse(isbn);
  259. match res {
  260. ParseResult::Valid(isbns, trail) => {
  261. assert_eq!(isbns.len(), 1);
  262. assert_eq!(isbns[0].text, isbn);
  263. assert_eq!(isbns[0].tags.len(), 0);
  264. assert_eq!(trail, "");
  265. },
  266. x => panic!("bad parse: {:?}", x)
  267. }
  268. }
  269. #[test]
  270. fn test_parse_isbn_trail() {
  271. let src = "349224010X :";
  272. let isbn = "349224010X";
  273. let defs = ParserDefs::new();
  274. let res = defs.parse(src);
  275. match res {
  276. ParseResult::Valid(isbns, trail) => {
  277. assert_eq!(isbns.len(), 1);
  278. assert_eq!(isbns[0].text, isbn);
  279. assert_eq!(isbns[0].tags.len(), 0);
  280. assert_eq!(trail, "");
  281. },
  282. x => panic!("bad parse: {:?}", x)
  283. }
  284. }
  285. #[test]
  286. fn test_scan_caron() {
  287. // this string has a combining mark (caron, unicode 730) in it
  288. let src = "349̌224010X";
  289. // we want a cleaned ISBN
  290. let isbn = "349224010X";
  291. let defs = ParserDefs::new();
  292. let mut parser = defs.create_parser(src);
  293. let res = parser.read_isbn().unwrap();
  294. assert_eq!(res.text, isbn);
  295. }
  296. #[test]
  297. fn test_parse_isbn_caron() {
  298. let src = "349̌224010X";
  299. let isbn = "349224010X";
  300. let defs = ParserDefs::new();
  301. let res = defs.parse(src);
  302. match res {
  303. ParseResult::Valid(isbns, trail) => {
  304. assert_eq!(isbns.len(), 1);
  305. assert_eq!(isbns[0].text, isbn);
  306. assert_eq!(isbns[0].tags.len(), 0);
  307. assert_eq!(trail, "");
  308. },
  309. x => panic!("bad parse: {:?}", x)
  310. }
  311. }
  312. #[test]
  313. fn test_parse_hyphen_isbn() {
  314. let src = "978-03-2948-9391";
  315. let isbn = "9780329489391";
  316. let defs = ParserDefs::new();
  317. let mut parser = defs.create_parser(src);
  318. let scan = parser.read_isbn();
  319. assert!(scan.is_some());
  320. let scan = scan.unwrap();
  321. assert_eq!(scan.text, isbn);
  322. assert_eq!(scan.tags.len(), 0);
  323. assert!(parser.is_empty());
  324. let res = defs.parse(src);
  325. match res {
  326. ParseResult::Valid(isbns, trail) => {
  327. assert_eq!(isbns.len(), 1);
  328. assert_eq!(isbns[0].text, isbn);
  329. assert_eq!(isbns[0].tags.len(), 0);
  330. assert_eq!(trail, "");
  331. },
  332. x => panic!("bad parse: {:?}", x)
  333. }
  334. }
  335. #[test]
  336. fn test_parse_space_isbn() {
  337. let src = "978 032948-9391";
  338. let isbn = "9780329489391";
  339. let defs = ParserDefs::new();
  340. let mut parser = defs.create_parser(src);
  341. let scan = parser.read_isbn();
  342. assert!(scan.is_some());
  343. let scan = scan.unwrap();
  344. assert_eq!(scan.text, isbn);
  345. assert_eq!(scan.tags.len(), 0);
  346. assert!(parser.is_empty());
  347. let res = defs.parse(src);
  348. match res {
  349. ParseResult::Valid(isbns, trail) => {
  350. assert_eq!(isbns.len(), 1);
  351. assert_eq!(isbns[0].text, isbn);
  352. assert_eq!(isbns[0].tags.len(), 0);
  353. assert_eq!(trail, "");
  354. },
  355. x => panic!("bad parse: {:?}", x)
  356. }
  357. }
  358. #[test]
  359. fn test_parse_isbn_tag() {
  360. let src = "34922401038 (set)";
  361. let isbn = "34922401038";
  362. let tag = "set";
  363. let defs = ParserDefs::new();
  364. let mut parser = defs.create_parser(src);
  365. let scan = parser.read_isbn();
  366. assert!(scan.is_some());
  367. let scan = scan.unwrap();
  368. assert_eq!(scan.text, isbn);
  369. assert_eq!(scan.tags, vec![tag]);
  370. assert!(parser.is_empty());
  371. let res = defs.parse(src);
  372. match res {
  373. ParseResult::Valid(isbns, trail) => {
  374. assert_eq!(isbns.len(), 1);
  375. assert_eq!(isbns[0].text, isbn);
  376. assert_eq!(isbns[0].tags, vec![tag]);
  377. assert_eq!(trail, "");
  378. },
  379. x => panic!("bad parse: {:?}", x)
  380. }
  381. }
  382. #[test]
  383. fn test_parse_isbn_square_tag() {
  384. let src = "34922401038 [set]";
  385. let isbn = "34922401038";
  386. let tag = "set";
  387. let defs = ParserDefs::new();
  388. let mut parser = defs.create_parser(src);
  389. let scan = parser.read_isbn();
  390. assert!(scan.is_some());
  391. let scan = scan.unwrap();
  392. assert_eq!(scan.text, isbn);
  393. assert_eq!(scan.tags, vec![tag]);
  394. assert!(parser.is_empty());
  395. let res = defs.parse(src);
  396. match res {
  397. ParseResult::Valid(isbns, trail) => {
  398. assert_eq!(isbns.len(), 1);
  399. assert_eq!(isbns[0].text, isbn);
  400. assert_eq!(isbns[0].tags, vec![tag]);
  401. assert_eq!(trail, "");
  402. },
  403. x => panic!("bad parse: {:?}", x)
  404. }
  405. }
  406. #[test]
  407. fn test_parse_isbn_multi_tag_sep() {
  408. let src = "34922401038 (set : alk. paper)";
  409. let isbn = "34922401038";
  410. let defs = ParserDefs::new();
  411. let mut parser = defs.create_parser(src);
  412. let scan = parser.read_isbn();
  413. assert!(scan.is_some());
  414. let scan = scan.unwrap();
  415. assert_eq!(scan.text, isbn);
  416. assert_eq!(scan.tags, vec!["set", "alk. paper"]);
  417. assert!(parser.is_empty());
  418. let res = defs.parse(src);
  419. match res {
  420. ParseResult::Valid(isbns, trail) => {
  421. assert_eq!(isbns.len(), 1);
  422. assert_eq!(isbns[0].text, isbn);
  423. assert_eq!(isbns[0].tags, vec!["set", "alk. paper"]);
  424. assert_eq!(trail, "");
  425. },
  426. x => panic!("bad parse: {:?}", x)
  427. }
  428. }
  429. #[test]
  430. fn test_parse_isbn_tags() {
  431. let src = "34922401038 (pbk.) (set)";
  432. let isbn = "34922401038";
  433. let defs = ParserDefs::new();
  434. let mut parser = defs.create_parser(src);
  435. let scan = parser.read_isbn();
  436. assert!(scan.is_some());
  437. let scan = scan.unwrap();
  438. assert_eq!(scan.text, isbn);
  439. assert_eq!(scan.tags, vec!["pbk.", "set"]);
  440. assert!(parser.is_empty());
  441. let res = defs.parse(src);
  442. match res {
  443. ParseResult::Valid(isbns, trail) => {
  444. assert_eq!(isbns.len(), 1);
  445. assert_eq!(isbns[0].text, isbn);
  446. assert_eq!(isbns[0].tags, vec!["pbk.", "set"]);
  447. assert_eq!(trail, "");
  448. },
  449. x => panic!("bad parse: {:?}", x)
  450. }
  451. }
  452. #[test]
  453. fn test_parse_isbn_leader() {
  454. let src = "a 970238408138";
  455. let isbn = "970238408138";
  456. let defs = ParserDefs::new();
  457. let mut parser = defs.create_parser(src);
  458. let scan = parser.read_isbn();
  459. assert!(scan.is_some());
  460. let scan = scan.unwrap();
  461. assert_eq!(scan.text, isbn);
  462. assert_eq!(scan.tags.len(), 0);
  463. assert!(parser.is_empty());
  464. let res = defs.parse(src);
  465. match res {
  466. ParseResult::Valid(isbns, trail) => {
  467. assert_eq!(isbns.len(), 1);
  468. assert_eq!(isbns[0].text, isbn);
  469. assert_eq!(isbns[0].tags.len(), 0);
  470. assert_eq!(trail, "");
  471. },
  472. x => panic!("bad parse: {:?}", x)
  473. }
  474. }
  475. #[test]
  476. fn test_parse_two_isbns_ws() {
  477. let src = "970238408138 30148100103";
  478. let isbn1 = "970238408138";
  479. let isbn2 = "30148100103";
  480. let defs = ParserDefs::new();
  481. let mut parser = defs.create_parser(src);
  482. let scan = parser.read_isbn();
  483. assert!(scan.is_some());
  484. let scan = scan.unwrap();
  485. assert_eq!(scan.text, isbn1);
  486. assert_eq!(scan.tags.len(), 0);
  487. assert_eq!(parser.position, isbn1.len());
  488. let res = defs.parse(src);
  489. match res {
  490. ParseResult::Valid(isbns, trail) => {
  491. assert_eq!(isbns.len(), 2);
  492. assert_eq!(isbns[0].text, isbn1);
  493. assert_eq!(isbns[0].tags.len(), 0);
  494. assert_eq!(isbns[1].text, isbn2);
  495. assert_eq!(isbns[1].tags.len(), 0);
  496. assert_eq!(trail, "");
  497. },
  498. x => panic!("bad parse: {:?}", x)
  499. }
  500. }
  501. #[test]
  502. fn test_parse_two_isbns_semi() {
  503. let src = "970238408138; ISBN 30148100103";
  504. let isbn1 = "970238408138";
  505. let isbn2 = "30148100103";
  506. let defs = ParserDefs::new();
  507. let mut parser = defs.create_parser(src);
  508. let scan = parser.read_isbn();
  509. assert!(scan.is_some());
  510. let scan = scan.unwrap();
  511. assert_eq!(scan.text, isbn1);
  512. assert_eq!(scan.tags.len(), 0);
  513. assert_eq!(parser.position, isbn1.len());
  514. let res = defs.parse(src);
  515. match res {
  516. ParseResult::Valid(isbns, trail) => {
  517. assert_eq!(isbns.len(), 2);
  518. assert_eq!(isbns[0].text, isbn1);
  519. assert_eq!(isbns[0].tags.len(), 0);
  520. assert_eq!(isbns[1].text, isbn2);
  521. assert_eq!(isbns[1].tags.len(), 0);
  522. assert_eq!(trail, "");
  523. },
  524. x => panic!("bad parse: {:?}", x)
  525. }
  526. }
  527. #[test]
  528. fn test_parse_two_isbns_real() {
  529. // Real example from record 2175696
  530. let src = "8719359022. ISBN 8719359004 (pbk.)";
  531. let isbn1 = "8719359022";
  532. let isbn2 = "8719359004";
  533. let defs = ParserDefs::new();
  534. let mut parser = defs.create_parser(src);
  535. let scan = parser.read_isbn();
  536. assert!(scan.is_some());
  537. let scan = scan.unwrap();
  538. assert_eq!(scan.text, isbn1);
  539. assert_eq!(scan.tags.len(), 0);
  540. let res = defs.parse(src);
  541. match res {
  542. ParseResult::Valid(isbns, trail) => {
  543. assert_eq!(isbns.len(), 2);
  544. assert_eq!(isbns[0].text, isbn1);
  545. assert_eq!(isbns[0].tags.len(), 0);
  546. assert_eq!(isbns[1].text, isbn2);
  547. assert_eq!(isbns[1].tags, vec!["pbk."]);
  548. assert_eq!(trail, "");
  549. },
  550. x => panic!("bad parse: {:?}", x)
  551. }
  552. }
Tip!

Press p or to see the previous file or, n or to see the next file