This is a library for training and applying sparse fine-tunings with torch and transformers.

conda create -n sft python=3.9
conda activate sft
conda install pytorch cudatoolkit=11.1 -c pytorch -c conda-forge

git clone https://github.com/cambridgeltl/composable-sft.git
cd composable-sft
pip install -e .

from transformers import AutoConfig, AutoModelForTokenClassification
from sft import SFT

config = AutoConfig.from_pretrained(
    'bert-base-multilingual-cased',
    num_labels=17,
)

model = AutoModelForTokenClassification.from_pretrained(
    'bert-base-multilingual-cased',
    config=config,
)

language_sft = SFT('cambridgeltl/mbert-lang-sft-bxr-small') # SFT for Buryat
task_sft = SFT('cambridgeltl/mbert-task-sft-pos') # SFT for POS tagging

# Apply SFTs to pre-trained mBERT TokenClassification model
language_sft.apply(model)
task_sft.apply(model)

@misc{ansell2021composable,
      title={Composable Sparse Fine-Tuning for Cross-Lingual Transfer},
      author={Alan Ansell and Edoardo Maria Ponti and Anna Korhonen and Ivan Vuli\'{c}},
      year={2021},
      eprint={2110.07560},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}