Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

data.py 5.4 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  1. import re
  2. from typing import Dict, List, Tuple
  3. import numpy as np
  4. import pandas as pd
  5. import ray
  6. from ray.data import Dataset
  7. from ray.data.preprocessor import Preprocessor
  8. from sklearn.model_selection import train_test_split
  9. from transformers import BertTokenizer
  10. from madewithml.config import STOPWORDS
  11. def load_data(dataset_loc: str, num_samples: int = None) -> Dataset:
  12. """Load data from source into a Ray Dataset.
  13. Args:
  14. dataset_loc (str): Location of the dataset.
  15. num_samples (int, optional): The number of samples to load. Defaults to None.
  16. Returns:
  17. Dataset: Our dataset represented by a Ray Dataset.
  18. """
  19. ds = ray.data.read_csv(dataset_loc)
  20. ds = ds.random_shuffle(seed=1234)
  21. ds = ray.data.from_items(ds.take(num_samples)) if num_samples else ds
  22. return ds
  23. def stratify_split(
  24. ds: Dataset,
  25. stratify: str,
  26. test_size: float,
  27. shuffle: bool = True,
  28. seed: int = 1234,
  29. ) -> Tuple[Dataset, Dataset]:
  30. """Split a dataset into train and test splits with equal
  31. amounts of data points from each class in the column we
  32. want to stratify on.
  33. Args:
  34. ds (Dataset): Input dataset to split.
  35. stratify (str): Name of column to split on.
  36. test_size (float): Proportion of dataset to split for test set.
  37. shuffle (bool, optional): whether to shuffle the dataset. Defaults to True.
  38. seed (int, optional): seed for shuffling. Defaults to 1234.
  39. Returns:
  40. Tuple[Dataset, Dataset]: the stratified train and test datasets.
  41. """
  42. def _add_split(df: pd.DataFrame) -> pd.DataFrame: # pragma: no cover, used in parent function
  43. """Naively split a dataframe into train and test splits.
  44. Add a column specifying whether it's the train or test split."""
  45. train, test = train_test_split(df, test_size=test_size, shuffle=shuffle, random_state=seed)
  46. train["_split"] = "train"
  47. test["_split"] = "test"
  48. return pd.concat([train, test])
  49. def _filter_split(df: pd.DataFrame, split: str) -> pd.DataFrame: # pragma: no cover, used in parent function
  50. """Filter by data points that match the split column's value
  51. and return the dataframe with the _split column dropped."""
  52. return df[df["_split"] == split].drop("_split", axis=1)
  53. # Train, test split with stratify
  54. grouped = ds.groupby(stratify).map_groups(_add_split, batch_format="pandas") # group by each unique value in the column we want to stratify on
  55. train_ds = grouped.map_batches(_filter_split, fn_kwargs={"split": "train"}, batch_format="pandas") # combine
  56. test_ds = grouped.map_batches(_filter_split, fn_kwargs={"split": "test"}, batch_format="pandas") # combine
  57. # Shuffle each split (required)
  58. train_ds = train_ds.random_shuffle(seed=seed)
  59. test_ds = test_ds.random_shuffle(seed=seed)
  60. return train_ds, test_ds
  61. def clean_text(text: str, stopwords: List = STOPWORDS) -> str:
  62. """Clean raw text string.
  63. Args:
  64. text (str): Raw text to clean.
  65. stopwords (List, optional): list of words to filter out. Defaults to STOPWORDS.
  66. Returns:
  67. str: cleaned text.
  68. """
  69. # Lower
  70. text = text.lower()
  71. # Remove stopwords
  72. pattern = re.compile(r"\b(" + r"|".join(stopwords) + r")\b\s*")
  73. text = pattern.sub(" ", text)
  74. # Spacing and filters
  75. text = re.sub(r"([!\"'#$%&()*\+,-./:;<=>?@\\\[\]^_`{|}~])", r" \1 ", text) # add spacing
  76. text = re.sub("[^A-Za-z0-9]+", " ", text) # remove non alphanumeric chars
  77. text = re.sub(" +", " ", text) # remove multiple spaces
  78. text = text.strip() # strip white space at the ends
  79. text = re.sub(r"http\S+", "", text) # remove links
  80. return text
  81. def tokenize(batch: Dict) -> Dict:
  82. """Tokenize the text input in our batch using a tokenizer.
  83. Args:
  84. batch (Dict): batch of data with the text inputs to tokenize.
  85. Returns:
  86. Dict: batch of data with the results of tokenization (`input_ids` and `attention_mask`) on the text inputs.
  87. """
  88. tokenizer = BertTokenizer.from_pretrained("allenai/scibert_scivocab_uncased", return_dict=False)
  89. encoded_inputs = tokenizer(batch["text"].tolist(), return_tensors="np", padding="longest")
  90. return dict(ids=encoded_inputs["input_ids"], masks=encoded_inputs["attention_mask"], targets=np.array(batch["tag"]))
  91. def preprocess(df: pd.DataFrame, class_to_index: Dict) -> Dict:
  92. """Preprocess the data in our dataframe.
  93. Args:
  94. df (pd.DataFrame): Raw dataframe to preprocess.
  95. class_to_index (Dict): Mapping of class names to indices.
  96. Returns:
  97. Dict: preprocessed data (ids, masks, targets).
  98. """
  99. df["text"] = df.title + " " + df.description # feature engineering
  100. df["text"] = df.text.apply(clean_text) # clean text
  101. df = df.drop(columns=["id", "created_on", "title", "description"], errors="ignore") # clean dataframe
  102. df = df[["text", "tag"]] # rearrange columns
  103. df["tag"] = df["tag"].map(class_to_index) # label encoding
  104. outputs = tokenize(df)
  105. return outputs
  106. class CustomPreprocessor(Preprocessor):
  107. """Custom preprocessor class."""
  108. def _fit(self, ds):
  109. tags = ds.unique(column="tag")
  110. self.class_to_index = {tag: i for i, tag in enumerate(tags)}
  111. self.index_to_class = {v: k for k, v in self.class_to_index.items()}
  112. def _transform_pandas(self, batch): # could also do _transform_numpy
  113. return preprocess(batch, class_to_index=self.class_to_index)
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...