Bookdata-tools/openlibrary/dvc.lock

schema: '2.0'
stages:
  scan-authors:
    cmd: cargo run --release -- openlib scan-authors ../data/openlib/ol_dump_authors.txt.gz
    deps:
    - path: ../data/openlib/ol_dump_authors.txt.gz
      hash: md5
      md5: d174f253b528f74c0d6446f38feb6b63
      size: 598486891
    - path: ../src/cli/openlib.rs
      md5: dbe8caca3345f0f9ca7f730c2df79f29
      size: 1938
    - path: ../src/openlib/
      hash: md5
      md5: 27c1c622ab641d042af5b8fe2685d1ff.dir
      size: 16669
      nfiles: 6
    outs:
    - path: author-names.parquet
      hash: md5
      md5: c83794d9417c6ed9730b55f8e4070aec
      size: 206500073
    - path: authors.parquet
      hash: md5
      md5: fe8c17f92d93dd3e85184927d3dff22e
      size: 240905689
  scan-editions:
    cmd: cargo run --release -- openlib scan-editions ../data/openlib/ol_dump_editions.txt.gz
    deps:
    - path: ../data/openlib/ol_dump_editions.txt.gz
      hash: md5
      md5: 6b13afe5a50fb1aeeb430bcd01164d13
      size: 10078352001
    - path: ../src/cli/openlib.rs
      md5: dbe8caca3345f0f9ca7f730c2df79f29
      size: 1938
    - path: ../src/openlib/
      hash: md5
      md5: 27c1c622ab641d042af5b8fe2685d1ff.dir
      size: 16669
      nfiles: 6
    - path: author-ids-after-works.parquet
      hash: md5
      md5: e83ffaacbe90cdfb864d4035b7c7a0f1
      size: 89062717
    - path: authors.parquet
      hash: md5
      md5: fe8c17f92d93dd3e85184927d3dff22e
      size: 240905689
    - path: works.parquet
      hash: md5
      md5: e142c29e5d077d22c20f27497b1dcd08
      size: 923755753
    outs:
    - path: all-authors.parquet
      hash: md5
      md5: 1bb21b70ab9dc309b4e4643323abf292
      size: 89082395
    - path: all-works.parquet
      hash: md5
      md5: e0b8264cf73b57d1d9d8ed8dc6339f85
      size: 242195062
    - path: edition-authors.parquet
      hash: md5
      md5: d829268d4697178064cdd961c348f0a5
      size: 288214520
    - path: edition-isbns.parquet
      hash: md5
      md5: 53ad906e9e4b3d41115df7a36a08f002
      size: 304629821
    - path: edition-subjects.parquet
      hash: md5
      md5: 269c73b599b9b833c9c6ba53eb75449d
      size: 653028313
    - path: edition-works.parquet
      hash: md5
      md5: e4d17ebeaf8d5a41ad6904324556af29
      size: 293781641
    - path: editions.parquet
      hash: md5
      md5: 742e29d6617f26b7696d790b3549fad5
      size: 1216690399
  scan-works:
    cmd: cargo run --release -- openlib scan-works ../data/openlib/ol_dump_works.txt.gz
    deps:
    - path: ../data/openlib/ol_dump_works.txt.gz
      hash: md5
      md5: 0d96507a98acf4dcc5416b10d85b1225
      size: 3200947685
    - path: ../src/cli/openlib.rs
      md5: dbe8caca3345f0f9ca7f730c2df79f29
      size: 1938
    - path: ../src/openlib/
      hash: md5
      md5: 27c1c622ab641d042af5b8fe2685d1ff.dir
      size: 16669
      nfiles: 6
    - path: authors.parquet
      hash: md5
      md5: fe8c17f92d93dd3e85184927d3dff22e
      size: 240905689
    outs:
    - path: author-ids-after-works.parquet
      hash: md5
      md5: e83ffaacbe90cdfb864d4035b7c7a0f1
      size: 89062717
    - path: work-authors.parquet
      hash: md5
      md5: f005bf244e8c22e2686f7ddbc4447bf3
      size: 225212340
    - path: work-subjects.parquet
      hash: md5
      md5: a7d4bf8eadf91625766eb638daed322a
      size: 543492486
    - path: works.parquet
      hash: md5
      md5: e142c29e5d077d22c20f27497b1dcd08
      size: 923755753
  edition-isbn-ids:
    cmd: cargo run --release -- link-isbn-ids -R edition -o openlibrary/edition-isbn-ids.parquet
      openlibrary/edition-isbns.parquet
    deps:
    - path: book-links/all-isbns.parquet
      hash: md5
      md5: 8803c162ab97efac8b098df7e9252314
      size: 464207536
    - path: openlibrary/edition-isbns.parquet
      hash: md5
      md5: 53ad906e9e4b3d41115df7a36a08f002
      size: 304629821
    outs:
    - path: openlibrary/edition-isbn-ids.parquet
      hash: md5
      md5: 348916ff4a045bd10dcf4ef8b6eb93e7
      size: 226887130
  schema@edition-isbn-ids:
    cmd: python ../run.py --rust pq-info -o edition-isbn-ids.json edition-isbn-ids.parquet
    deps:
    - path: edition-isbn-ids.parquet
      md5: 6cf6500ebab273315fa90e2f2031dd3b
      size: 145485299
    outs:
    - path: edition-isbn-ids.json
      md5: 0fe837f400c6e7b7d4535b67237af989
      size: 251
  schema@works:
    cmd: python ../run.py --rust pq-info -o works.json works.parquet
    deps:
    - path: works.parquet
      md5: 0392b4f5308bf945efc14d206bb8b9fa
      size: 663422438
    outs:
    - path: works.json
      md5: 5b5e973797c2f0d8aff51c7f6d300fa6
      size: 327
  schema@all-works:
    cmd: python ../run.py --rust pq-info -o all-works.json all-works.parquet
    deps:
    - path: all-works.parquet
      md5: 832cc1253307df9c7bbe4d83e7e3fa93
      size: 176243794
    outs:
    - path: all-works.json
      md5: 1e5228092f43d81e5871c48117be3146
      size: 239
  schema@editions:
    cmd: python ../run.py --rust pq-info -o editions.json editions.parquet
    deps:
    - path: editions.parquet
      md5: b082da165d2b2d9c0fa5489376c8c57a
      size: 888704794
    outs:
    - path: editions.json
      md5: 464dc59fc293103fcbf3e9baa3373c1a
      size: 327
  schema@work-authors:
    cmd: python ../run.py --rust pq-info -o work-authors.json work-authors.parquet
    deps:
    - path: work-authors.parquet
      md5: e62c8dfb5507c4ec67b3e0f7b940523f
      size: 153899196
    outs:
    - path: work-authors.json
      md5: 140a8f1c6dc0ad558bf6ca19e742210f
      size: 331
  schema@author-names:
    cmd: python ../run.py --rust pq-info -o author-names.json author-names.parquet
    deps:
    - path: author-names.parquet
      md5: 76ca0c2e90393598cc71578ffe5493d1
      size: 151081619
    outs:
    - path: author-names.json
      md5: a8b5bf82ca7a88ff7ef58ec4698a0e3a
      size: 331
  schema@edition-authors:
    cmd: python ../run.py --rust pq-info -o edition-authors.json edition-authors.parquet
    deps:
    - path: edition-authors.parquet
      md5: 231cda3e2643ce16e46942a58aa3b57e
      size: 193584177
    outs:
    - path: edition-authors.json
      md5: 4ba99be63216163455443c29e8acafe4
      size: 336
  schema@edition-isbns:
    cmd: python ../run.py --rust pq-info -o edition-isbns.json edition-isbns.parquet
    deps:
    - path: edition-isbns.parquet
      md5: 4e25efdddefdaa656de713a99b0ca2d7
      size: 207849845
    outs:
    - path: edition-isbns.json
      md5: 7f750ebd2dcdfebb39eb49c0cf6948a2
      size: 247
  schema@edition-works:
    cmd: python ../run.py --rust pq-info -o edition-works.json edition-works.parquet
    deps:
    - path: edition-works.parquet
      md5: 9e7de4af3d400505f77235d8f903f826
      size: 203839441
    outs:
    - path: edition-works.json
      md5: d8b6f877272dbc1c7f690a5774ddada8
      size: 248
  schema@all-authors:
    cmd: python ../run.py --rust pq-info -o all-authors.json all-authors.parquet
    deps:
    - path: all-authors.parquet
      md5: 632add46e59ff1653d9532ecbc87b851
      size: 67096542
    outs:
    - path: all-authors.json
      md5: 6921aee9159edf48ca9a733245113f6e
      size: 237
  schema@authors:
    cmd: python ../run.py --rust pq-info -o authors.json authors.parquet
    deps:
    - path: authors.parquet
      md5: c77ceaa2f2cb3f90f514edd561e11398
      size: 178808160
    outs:
    - path: authors.json
      md5: a33ba4a8bb933055f43ec0d933cdebec
      size: 325
  schema@work-subjects:
    cmd: python ../run.py --rust pq-info -o work-subjects.json work-subjects.parquet
    deps:
    - path: work-subjects.parquet
      md5: 273c91a6d0b49f57ae8d10816fb4f5a0
      size: 304767903
    outs:
    - path: work-subjects.json
      md5: 164a92469b9c94c2d99ff5fa8ab27551
      size: 245
  schema@edition-subjects:
    cmd: python ../run.py --rust pq-info -o edition-subjects.json edition-subjects.parquet
    deps:
    - path: edition-subjects.parquet
      md5: d58f4b70cf2a7ebf8d919aa18113d2e2
      size: 522800217
    outs:
    - path: edition-subjects.json
      md5: 04631619a35917b9dd4a850541190b8d
      size: 245
  work-clusters:
    cmd: cargo run --release -- cluster extract-books -n work_id -o openlibrary/work-clusters.parquet
      OL-W
    deps:
    - path: book-links/cluster-graph-nodes.parquet
      hash: md5
      md5: 2f0e64cd13b40c850326a75d4e69731a
      size: 1226145163
    outs:
    - path: openlibrary/work-clusters.parquet
      hash: md5
      md5: 95cb00013c55edf3bfea1dcc703f32d2
      size: 238932284
  schema@work-clusters:
    cmd: python ../run.py --rust pq-info -o work-clusters.json work-clusters.parquet
    deps:
    - path: work-clusters.parquet
      md5: 82b15db1b6cb8a76cd9526a1770bc8dc
      size: 172722261
    outs:
    - path: work-clusters.json
      md5: 9b31d738006f09d80ffc1e2da748d976
      size: 251