Commit 5ee1dd4b authored by Benjamin Murauer's avatar Benjamin Murauer
Browse files

removed unneeded method

parent ba1b5def
Pipeline #51150 failed with stage
in 1 minute and 26 seconds
"""Loaders that provide explicit splits for train/testing."""
from abc import abstractmethod
import os
import random
from typing import List, Optional, Tuple
......@@ -12,16 +11,6 @@ from sklearn.model_selection import StratifiedKFold
from dbispipeline.base import Loader
def _load_df(path: str) -> pd.DataFrame:
df = pd.read_csv(os.path.join(path, 'dataset.csv'))
unnamed_columns = [c for c in df.columns if c.startswith('Unnamed')]
if unnamed_columns:
df = df.drop(columns=unnamed_columns)
df['text_raw'] = [os.path.join(path, x) for x in df.text_raw]
df['stanza'] = [os.path.join(path, x) for x in df.stanza]
return df
def _attach(df: pd.DataFrame, y: np.array) -> Tuple[pd.DataFrame, str]:
df = df.copy()
# attach the column to the dataframe for grouping
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment