Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

sources.rs 1.6 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  1. use std::io::prelude::*;
  2. use std::io::Lines;
  3. use anyhow::Result;
  4. use postgres::rows::LazyRows;
  5. use fallible_iterator::FallibleIterator;
  6. use log::*;
  7. use crate::tsv::split_first;
  8. use super::parsers::*;
  9. pub type IdPR = (i64, ParseResult);
  10. pub struct FileSource<B> {
  11. parsers: ParserDefs,
  12. lines: Lines<B>
  13. }
  14. impl <B: BufRead> FileSource<B> {
  15. pub fn create(read: B) -> Result<FileSource<B>> {
  16. Ok(FileSource {
  17. parsers: ParserDefs::new(),
  18. lines: read.lines()
  19. })
  20. }
  21. }
  22. impl <B: BufRead> FallibleIterator for FileSource<B> {
  23. type Item = IdPR;
  24. type Error = anyhow::Error;
  25. fn next(&mut self) -> Result<Option<IdPR>> {
  26. let nl = self.lines.next();
  27. match nl {
  28. None => Ok(None),
  29. Some(line) => {
  30. let text = line?;
  31. let (id, isbn) = split_first(&text).unwrap();
  32. Ok(Some((id.parse::<i64>()?, self.parsers.parse(isbn))))
  33. }
  34. }
  35. }
  36. }
  37. pub struct DBSource<'t, 's> {
  38. parsers: ParserDefs,
  39. rows: LazyRows<'t, 's>
  40. }
  41. impl <'t, 's> DBSource<'t, 's> {
  42. pub fn create(rows: LazyRows<'t, 's>) -> Result<DBSource<'t, 's>> {
  43. Ok(DBSource {
  44. parsers: ParserDefs::new(),
  45. rows: rows
  46. })
  47. }
  48. }
  49. impl <'t, 's> FallibleIterator for DBSource<'t, 's> {
  50. type Item = IdPR;
  51. type Error = anyhow::Error;
  52. fn next(&mut self) -> Result<Option<IdPR>> {
  53. if let Some(row) = self.rows.next()? {
  54. let id: i32 = row.get(0);
  55. let content: String = row.get(1);
  56. let result = self.parsers.parse(&content);
  57. debug!("{}: {:?}", id, result);
  58. Ok(Some((id.into(), result)))
  59. } else {
  60. Ok(None)
  61. }
  62. }
  63. }
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...