Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

sinks.rs 1.8 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  1. use std::io::prelude::*;
  2. use anyhow::Result;
  3. use sha1::Sha1;
  4. use super::parsers::ISBN;
  5. use crate::db::{ConnectInfo, CopyRequest, CopyTarget};
  6. use crate::cleaning::write_pgencoded;
  7. pub trait WriteISBNs {
  8. fn write_isbn(&mut self, id: i64, isbn: &ISBN) -> Result<()>;
  9. fn finish(&mut self) -> Result<String> {
  10. Ok("".to_owned())
  11. }
  12. }
  13. pub struct NullWriter {}
  14. impl WriteISBNs for NullWriter {
  15. fn write_isbn(&mut self, _id: i64, _isbn: &ISBN) -> Result<()> {
  16. Ok(())
  17. }
  18. }
  19. pub struct FileWriter {
  20. pub write: Box<dyn Write>
  21. }
  22. impl WriteISBNs for FileWriter {
  23. fn write_isbn(&mut self, id: i64, isbn: &ISBN) -> Result<()> {
  24. if isbn.tags.len() > 0 {
  25. for tag in &isbn.tags {
  26. writeln!(self.write, "{}\t{}\t{}", id, isbn.text, tag)?;
  27. }
  28. } else {
  29. writeln!(self.write, "{}\t{}\t", id, isbn.text)?;
  30. }
  31. Ok(())
  32. }
  33. }
  34. pub struct DBWriter {
  35. hash: Sha1,
  36. target: CopyTarget
  37. }
  38. impl DBWriter {
  39. pub fn new<C: ConnectInfo>(db: &C, table: &str) -> Result<DBWriter> {
  40. let req = CopyRequest::new(db, table)?;
  41. let req = req.truncate(true);
  42. Ok(DBWriter {
  43. hash: Sha1::new(),
  44. target: req.open()?
  45. })
  46. }
  47. }
  48. impl WriteISBNs for DBWriter {
  49. fn write_isbn(&mut self, id: i64, isbn: &ISBN) -> Result<()> {
  50. self.hash.update(id.to_string().as_bytes());
  51. self.hash.update(isbn.text.as_bytes());
  52. if isbn.tags.len() > 0 {
  53. for tag in &isbn.tags {
  54. self.hash.update(tag.as_bytes());
  55. write!(self.target, "{}\t{}\t", id, isbn.text)?;
  56. write_pgencoded(&mut self.target, tag.as_bytes())?;
  57. writeln!(self.target)?;
  58. }
  59. } else {
  60. writeln!(self.target, "{}\t{}\t", id, isbn.text)?;
  61. }
  62. Ok(())
  63. }
  64. fn finish(&mut self) -> Result<String> {
  65. let hash = self.hash.hexdigest();
  66. Ok(hash.to_owned())
  67. }
  68. }
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...