get_assets/project.yml

title: "Pre-annotate data with Regex"
description: "In this project we pre-annotate  some data (labdocs) with regex patterns, thanks to Spacy, in order to correct them  with Label Studio. The resulting annotated dataset is used (in other project) train a Spacy Ner model "

# =======================================vars=======================================================================================
vars:
  # A number of sample tex labdocs to annotate
  user : "root"
  password : 11950022
  host : "localhost"
  database : "Labnbook"
  sample_size : 500
  train_size : 0.75
  # The name of the jsonl file of the text labdocs
  name : "labdoc_init"
  lang: "fr"
  #regex patterns for cleaning text labdocs 
  regex_text: "regex_text.json" 
  #regex patterns for sampling labdocs  labels
  regex_ner : "regex_ner.json"
  train: "train"
  dev: "dev"


# =======================================directories=======================================================================================
directories: ["source","scripts","data","configs"]
# Files that should be available in the directory
configs:
  - dest: "configs/${vars.regex_ner}"
    description: "Regex patterns for annotate labels"
  - dest: "configs/${vars.regex_text}"
    description: "Regex patterns for clean text"
# ============================================workflows=======================================================================================
workflows:
  get_labdocs:
    - get_init
    - get_sample
    - get_train
 # ============================================commands====================================================================================
commands:
  - name: "get_init"
    help: "Get all cleaned text Labdocs according to the regex patterns ${vars.regex_text}. We extract one Labdoc from each report. See the 'clean_text' function for more details about the applied clean."
    script:
        - "python3 scripts/get_init.py ${vars.user} ${vars.host} ${vars.database} ${vars.password}   ${vars.name} configs/${vars.regex_text}"
    outputs:
        - "source/${vars.name}.jsonl"
    deps:
        - "configs/${vars.regex_text}"
## ----------------------------------------------------------------------------------------------------------------------
  - name: "get_sample"
    help: "Get a sample of Labdocs with equations and table "
    script:
        - "python3 scripts/get_sample.py source/${vars.name}.jsonl  source/${vars.name}_sample.jsonl   ${vars.sample_size} configs/${vars.regex_ner}"
    outputs:
        - "source/${vars.name}_sample.jsonl"
    deps: 
        - "source/${vars.name}.jsonl"
## ----------------------------------------------------------------------------------------------------------------------      
  - name: "get_train"
    help: "Get a sample of Labdocs according to the Regex patterns in ${vars.regex_ner} for training and testing a NER model from scratch"
    script:
      - "python3 scripts/get_train.py  ${vars.lang}  source/${vars.name}_sample.jsonl    ${vars.train_size} configs/${vars.regex_ner}"
    outputs:
      # - data/${vars.dev}.json
      # - data/${vars.train}.json
      # - data/${vars.dev}.spacy
      # - data/${vars.dev}.spacy
      - data/all.spacy
      - data/all.json


    deps: 
      - "source/${vars.name}_sample.jsonl"