veld_code__nlp_generic_preprocessing/veld_preprocess_remove_punctuation.yaml at main · veldhub/veld_code__nlp_generic_preprocessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# TODO: update metadata
x-veld:
  code:
    description: "removes punctuation from text with spaCy pretrained models"
    topic:
      - "NLP"
      - "Preprocessing"
      - "ETL"

    input:
      - volume: /veld/input/
        environment_var: in_txt_file
        file_type: "txt"
        content: "raw text"

    output:
      - volume: /veld/output/txt/
        environment_var: out_txt_file
        file_type: "txt"
        content: "raw text"
      - volume: /veld/output/tmp/
        file_type: "txt"
        content: "raw text"

    config:
      - environment_var: out_data_description
        description: "automatic data description for generating a data veld yaml file"
        var_type: "str"
      - environment_var: cpu_count
        description: "number of cpu cores allocated to this processing. Defaults to maximum number
          of available cores"
        var_type: "int"
      - environment_var: buffer_segments
        description: "percentage of segments where processing results are persisted in between. So
          that processing could continue should it have crashed"
        var_type: "int"
        default: 100
      - environment_var: sleep_duration
        description: "number of seceonds between each multiprocess invokation, since with big data,
          a memory race condition can occurr. To work-around this, a small waiting period in between
          can be set with this variable."
        var_type: "int"
        default: 10

services:
  veld_preprocess_remove_punctuation:
    build: .
    volumes:
      - ./data/storage/spacy_cache/:/veld/storage/spacy_cache/
      - ./src/:/veld/code/:z
    command: python -u /veld/code/preprocess.py remove_punctuation
    environment:
      in_file: null
      out_file: null
      out_data_description: null
      spacy_model: null
      cpu_count: null
      buffer_size: 1