未命名.ipynb
13.9 KB
!head -2 data/x-stance/questions.en.json
head: error reading 'data/x-stance/': Is a directory
In [42]:
!ls data/x-stance
Out [42]:
questions.de.jsonl questions.en.jsonl questions.fr.jsonl questions.it.jsonl
In [16]:
import json import pandas as pd import os
In [48]:
json_list = [] for line in open('data/x-stance/questions.it.jsonl'): json_list.append(json.loads(line)) data = pd.DataFrame.from_dict(json_list)
In [37]:
k = 100 seed_list = [100,13,21,42,87]
In [39]:
!ls /home/mist/projects/LM-BFF/data/k-shot/x-stance
Out [39]:
100-100 100-13 100-21 100-42 100-87 16-100 16-13 16-21 16-42 16-87
In [23]:
output_dir = '/home/mist/projects/LM-BFF/data/k-shot/'
In [14]:
task = 'x-stance'
In [38]:
for seed in seed_list: task_dir = os.path.join(output_dir, task) setting_dir = os.path.join(task_dir, f"{k}-{seed}") os.makedirs(setting_dir, exist_ok=True) data.sample(random_state=seed,n=k)
In [45]:
data
| id | text | topic | |
|---|---|---|---|
| 0 | 2 | Finden Sie es grundsätzlich richtig, dass der ... | Welfare |
| 1 | 4 | Soll zusätzlich zur bestehenden Mutterschaftsv... | Welfare |
| 2 | 6 | Die Invalidenversicherung spricht bei nicht ob... | Welfare |
| 3 | 7 | Würden Sie eine nationale Spitalplanung befürw... | Healthcare |
| 4 | 9 | Finden Sie es richtig, dass einzelne ärztliche... | Healthcare |
| ... | ... | ... | ... |
| 189 | 3464 | Würden Sie eine Ausdehnung der rechtlichen Mög... | Security |
| 190 | 3468 | Soll die Schweiz Verhandlungen über den Beitri... | Foreign Policy |
| 191 | 3469 | Soll der Bundesrat ein Freihandelsabkommen mit... | Foreign Policy |
| 192 | 3470 | Eine Initiative fordert, dass die Haftungsrege... | Foreign Policy |
| 193 | 3471 | Befürworten Sie die Kandidatur der Schweiz für... | Foreign Policy |
194 rows × 3 columns
In [49]:
data['topic'].value_counts()
Out [49]:
Infrastructure & Environment 31 Economy 23 Security 20 Immigration 19 Society 17 Education 16 Foreign Policy 16 Finances 15 Welfare 15 Healthcare 11 Political System 9 Digitisation 2 Name: topic, dtype: int64
In [16]:
for seed in seed_list: data.sample(random_state=seed,n=k)
In [34]:
label_encoding = { 'Infrastructure & Environment': 0, 'Economy': 1 , 'Security': 2 , 'Immigration': 3 , 'Society': 4 , 'Education': 5 , 'Foreign Policy': 6 , 'Finances': 7 , 'Welfare':8 , 'Healthcare':9 , 'Political System': 10 , 'Digitisation':11 }
In [35]:
data['label'] = data['topic'].apply(lambda x:label_encoding[x])
In [40]:
data.shape
Out [40]:
(194, 4)
In [4]:
data['label']=data.apply(lambda x:'+1' if x['stars']>=3 else '-1',axis=1) data = data.sample(frac=1) data['text']=data['text'].apply(lambda x:' '.join(x.replace('\n','.').split(' ')[:500]))
In [9]:
data