<TBD>

git clone https://github.com/DFKI-NLP/REval
cd REval
pip install -r requirements.txt

python reval.py generate-all-from-semeval \
    --train-file <SEMEVAL DIR>/train.json \
    --validation-file <SEMEVAL DIR>/dev.json \
    --test-file <SEMEVAL DIR>/test.json \
    --output-dir ./data/semeval/

python reval.py generate-all-from-tacred \
    --train-file <TACRED DIR>/train.json \
    --validation-file <TACRED DIR>/dev.json \
    --test-file <TACRED DIR>/test.json \
    --output-dir ./data/tacred/

mkdir -p models/cnn_semeval
wget --content-disposition https://cloud.dfki.de/owncloud/index.php/s/F3gf9xkeb2foTFe/download -P models/cnn_semeval

python probing_task_evaluation.py \
    --model-dir ./models/cnn_semeval/ \
    --data-dir ./data/semeval/ \
    --dataset semeval2010 \
    --cuda-device 0 \
    --batch-size 64 \
    --cache-representations

{
    "ArgTypeHead": {
        "acc": 75.82,
        "devacc": 78.96,
        "ndev": 670,
        "ntest": 2283
    },
    "ArgTypeTail": {
        "acc": 75.4,
        "devacc": 78.79,
        "ndev": 627,
        "ntest": 2130
    },
    [...]
}

@inproceedings{alt-etal-2020-probing,
    title={Probing Linguistic Features of Sentence-level Representations in Neural Relation Extraction},
    author={Christoph Alt and Aleksandra Gabryszak and Leonhard Hennig},
    year={2020},
    booktitle={Proceedings of ACL},
    url={https://arxiv.org/abs/2004.08134}
}