Initial commit

{\rtf1\ansi\ansicpg1252\cocoartf1038\cocoasubrtf350

{\fonttbl\f0\fswiss\fcharset0 Helvetica;}

{\colortbl;\red255\green255\blue255;}

\margl1440\margr1440\vieww9000\viewh8400\viewkind0

\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural\pardirnatural

\f0\fs24 \cf0 No work files for this lesson...}

 No newline at end of file

%% Cell type:code id: tags:

``` python

import warnings 

warnings.filterwarnings('ignore')

def get_materials(spacy = False, networkx = False):

    if networkx:

        !pip uninstall networkx -yy

        !pip uninstall pydot -yy

        !sudo apt-get purge graphviz -yy

        !pip uninstall pyparsing -yy

        !pip install networkx

        !pip install pydotplus

        !sudo apt-get install graphviz -yy

    if spacy:   

        !pip install spacy

        !python -m spacy.en.download

#import sys

#sys.path.append('/home/jupyter/site-packages/')

from IPython.display import HTML

get_materials(spacy=True, networkx=True)

```

%% Output

Cannot uninstall requirement networkx, not installed

Cannot uninstall requirement pydot, not installed

Reading package lists... Done

Building dependency tree       

Reading state information... Done

The following packages were automatically installed and are no longer required:

  fonts-liberation libcdt5 libcgraph6 libgd3 libgvc6 libgvpr2 libltdl7

  libpathplan4 libvpx3

Use 'sudo apt autoremove' to remove them.

The following packages will be REMOVED:

  graphviz*

0 upgraded, 0 newly installed, 1 to remove and 70 not upgraded.

After this operation, 9,736 kB disk space will be freed.

(Reading database ... 29552 files and directories currently installed.)

Removing graphviz (2.38.0-12ubuntu2.1) ...

Purging configuration files for graphviz (2.38.0-12ubuntu2.1) ...

Processing triggers for man-db (2.7.5-1) ...

Uninstalling pyparsing-2.1.10:

  Successfully uninstalled pyparsing-2.1.10

Collecting networkx

  Using cached networkx-1.11-py2.py3-none-any.whl

Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python2.7/dist-packages (from networkx)

Installing collected packages: networkx

Successfully installed networkx-1.11

Collecting pydotplus

Requirement already satisfied: pyparsing>=2.0.1 in /usr/local/lib/python2.7/dist-packages (from pydotplus)

Installing collected packages: pydotplus

Successfully installed pydotplus-2.0.2

Reading package lists... Done

Building dependency tree       

Reading state information... Done

Suggested packages:

  gsfonts graphviz-doc

The following NEW packages will be installed:

  graphviz

0 upgraded, 1 newly installed, 0 to remove and 70 not upgraded.

Need to get 680 kB of archives.

After this operation, 9,736 kB of additional disk space will be used.

Get:1 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 graphviz amd64 2.38.0-12ubuntu2.1 [680 kB]

Fetched 680 kB in 1s (602 kB/s)

debconf: delaying package configuration, since apt-utils is not installed

Selecting previously unselected package graphviz.

(Reading database ... 29452 files and directories currently installed.)

Preparing to unpack .../graphviz_2.38.0-12ubuntu2.1_amd64.deb ...

Unpacking graphviz (2.38.0-12ubuntu2.1) ...

Processing triggers for man-db (2.7.5-1) ...

Setting up graphviz (2.38.0-12ubuntu2.1) ...

%% Cell type:code id: tags:

``` python

from pathlib import Path

import io

import json

import random

import matplotlib.pyplot as plt

import networkx as nx

import spacy.util

from spacy.en import English

from spacy.gold import GoldParse

from spacy.syntax.util import Config

from spacy.syntax.arc_eager import ArcEager

from spacy.syntax.parser import Parser

from spacy.scorer import Scorer

from spacy.tagger import Tagger

from spacy.pipeline import DependencyParser

from spacy.syntax.nonproj import PseudoProjectivity

from spacy.language import Language

from spacy.tokens import Doc

from spacy.vocab import Vocab

from spacy.symbols import PUNCT, SYM, VERB, DET, ADJ, NOUN, ADP, INTJ, X, PART, PRON, ADV, PROPN, CONJ

from spacy.attrs import POS

def prepare_environment(path_to_model):

    model_path = Path(path_to_model)

    if not model_path.exists():

        model_path.mkdir()

    if not (model_path / "deps").exists():

        (model_path / "deps").mkdir()

    if not (model_path / "pos").exists():

        (model_path / "pos").mkdir()             

def _parse_line(line):

    pieces = line.split()

    id_ = int(pieces[0].split('_')[-1])

    word = pieces[1]

    pos = pieces[4]

    head_idx = int(pieces[6])-1

    label = pieces[7]

    return word, pos, head_idx, label

def read_conll_from_path(path):

    """Read a standard CoNLL/MALT-style format"""

    with io.open(path, 'r', encoding='utf8') as file_:

        sents = []

        for sent_str in file_.read().strip().split('\n\n'):

            ids = []

            words = []

            heads = []

            labels = []

            tags = []

            for i, line in enumerate(sent_str.split('\n')):

                word, pos_string, head_idx, label = _parse_line(line)

                words.append(word)

                if head_idx < 0:

                    head_idx = i

                ids.append(i)

                heads.append(head_idx)

                labels.append(label)

                tags.append(pos_string)

            text = ' '.join(words)

            annot = (ids, words, tags, heads, labels, ['O'] * len(ids))

            sents.append((text, [(annot, [])]))

    return sents

def process_data(data):

    return PseudoProjectivity.preprocess_training_data(data)

def write_config_file(model_dir, features, actions):

    model_path = Path(model_dir)

    with (model_path / 'deps' / 'config.json').open('wb') as file_:

        file_.write(

            json.dumps(

                {'pseudoprojective': True, 'features': features, 'labels':actions}).encode('utf8'))

def get_label_set(gold_tuples):

    left_labels, right_labels = [set()] * 2

    for _, sents in gold_tuples:

        for annot_tuples, _ in sents:

            _, words, deps, heads, thing1, thing2 = annot_tuples

            for i, (head, dep) in enumerate(zip(heads, deps)):

                if i < head:

                    left_labels.add(dep)

                elif i > head:

                    right_labels.add(dep)   

    return sorted(left_labels), sorted(right_labels)

def create_language_model(model_dir, features, train_sents, actions):

    left_labels, right_labels = get_label_set(train_sents)

    tag_map = English.Defaults.tag_map

    additional_tags = {

        u'~':{POS:X},

        u'@':{POS:X},

        u'D':{POS: DET},

        u'A':{POS: ADJ}, 

        u'N':{POS: NOUN},  

        u'P':{POS: ADP},      

        u'!':{POS: INTJ},  

        u'#':{POS: X},   

        u',':{POS: PUNCT},    

        u'S':{POS: NOUN, u'Poss': u'yes'}, 

        u'Z':{POS: PROPN, u'Poss': u'yes'},

        u'O':{POS: PRON},

        u'R':{POS: ADV},        

        u'V':{POS:VERB},

        u'E':{POS: X},

        u'G':{POS: X},        

        u'^':{POS: PROPN},   

        u'&':{POS: CONJ},        

        u'L':{POS: VERB},  

        u'U':{POS: X},

        u'X':{POS: ADJ},

        u'T':{POS: PART},

        u'Y':{POS: VERB},

        u'M':{POS: VERB, u'NounType': u'prop'},

    }

    tag_map.update(additional_tags)

    vocab = Vocab(lex_attr_getters=Language.Defaults.lex_attr_getters, tag_map=tag_map)

    tagger = Tagger(vocab, tag_map=tag_map)

    parser = DependencyParser(vocab, features = features, left_labels=left_labels, right_labels=right_labels, actions = actions)    

    nlp = Language(vocab=vocab, tagger = tagger, parser = parser)

    return nlp

def score_model(nlp, gold_docs, verbose=False):

    scorer = Scorer()

    for _, gold_doc in gold_docs:

        for (ids, words, pos, dep_idx, dep_relations, entities), _ in gold_doc:

            doc = Doc(nlp.vocab, words=words)

            nlp.tagger(doc)

            nlp.parser(doc)

            PseudoProjectivity.deprojectivize(doc)

            gold = GoldParse(doc, tags=pos, heads=dep_idx, deps=dep_relations)

            scorer.score(doc, gold, verbose=verbose)

    return scorer

def train(nlp, training_sents,testing_sents, n_iter = 20):

    print("Itn.\t\tPOS% \tUAS% \tLAS%")

    scorer = score_model(nlp, testing_sents)

    print('%s:\t%d\t%.3f\t%.3f\t' % ("Pretrain", scorer.tags_acc, scorer.uas,scorer.las))

    for itn in range(n_iter): 

        loss = 0

        for _, sents in training_sents:

            for (ids, words, pos, dep_idx, dep_relations, entities), _ in sents:

                doc = nlp.tokenizer.tokens_from_list(words)

                gold = GoldParse(doc, annot_tuples=None, words=words, tags=pos, heads=dep_idx, deps=dep_relations, make_projective=False)                          

                nlp.tagger.update(doc, gold)

                nlp.tagger(doc)

                nlp.parser.update(doc, gold) 

        scorer = score_model(nlp, testing_sents)

        random.shuffle(training_sents)

        print('%d:\t\t%d\t%.3f\t%.3f\t' % (itn, scorer.tags_acc, scorer.uas,scorer.las))

    return nlp

```

%% Cell type:code id: tags:

``` python

def download_data():

    if not os.path.exists('Tweebo'):

        !git clone https://github.com/SowaLabs/Tweebo.git  

    train_path = 'Tweebo/Tweebank/Train_Test_Splited/train'

    test_path = 'Tweebo/Tweebank/Train_Test_Splited/test'

    return train_path, test_path

model_dir = 'custom_model/'

path_to_training_data, path_to_testing_data = download_data()

train_data = read_conll_from_path(path_to_training_data)

train_sents = process_data(train_data)

```

%% Output

Cloning into 'Tweebo'...

remote: Counting objects: 858, done.[K

remote: Compressing objects: 100% (643/643), done.[K

remote: Total 858 (delta 198), reused 858 (delta 198), pack-reused 0[K

Receiving objects: 100% (858/858), 18.89 MiB | 11.40 MiB/s, done.

Resolving deltas: 100% (198/198), done.

Checking connectivity... done.

%% Cell type:code id: tags:

``` python

for i in [8, 18, 43, 46, 56]:

    print train_sents[i][0]

    print

```

%% Output

@senorsarita What's that !

RT @thickinmyhips i hate being employed sometimes !

@monaeebbyy2k11 I miss you too <3 I miss |California| but imma come see you as soon as I come back I #promise

Stupid Kilkenny didn't get to meet @Royseven :/

Owning Your Craft in the Fitness Business - http://t.co/hTizU4q9 #Fitness BodyBuilding #Professionals #Vancouver

%% Cell type:markdown id: tags:

### CoNLL Format:

<br>

* ID: Word index, integer starting at 1 for each new sentence; may be a range for tokens with multiple words.

* FORM: Word form or punctuation symbol.

* LEMMA: Lemma or stem of word form.

* UPOSTAG: Universal part-of-speech tag drawn from our revised version of the Google universal POS tags.

* XPOSTAG: Language-specific part-of-speech tag; underscore if not available.

* FEATS: List of morphological features from the universal feature inventory or from a defined language-specific extension; underscore if not available.

* HEAD: Head of the current token, which is either a value of ID or zero (0).

* DEPREL: Universal Stanford dependency relation to the HEAD (root iff HEAD = 0) or a defined language-specific subtype of one.

* DEPS: List of secondary dependencies (head-deprel pairs).

* MISC: Any other annotation.

%% Cell type:code id: tags:

``` python

def train_tagger_and_parser_model(model_dir, path_to_training_data, path_to_testing_data):

    prepare_environment(model_dir)

    train_data = read_conll_from_path(path_to_training_data)

    train_sents = process_data(train_data)

    test_data = read_conll_from_path(path_to_testing_data)

    test_sents = process_data(test_data)

    actions = ArcEager.get_actions(gold_parses=train_sents)

    features = Language.Defaults.parser_features

    write_config_file(model_dir, features, actions)

    nlp = create_language_model(model_dir, features, train_sents, actions)

    nlp = train(nlp, train_sents, test_sents)

    return nlp

```

%% Cell type:code id: tags:

``` python

nlp = train_tagger_and_parser_model(model_dir, path_to_training_data, path_to_testing_data)

```

%% Output

Itn.		POS% 	UAS% 	LAS%

Pretrain:	2	26.488	5.706	

0:		71	67.172	63.966	

1:		75	65.868	62.416	

2:		75	69.215	65.868	

3:		75	71.786	67.524	

4:		78	71.786	68.897	

5:		81	73.300	69.602	

6:		78	75.308	72.032	

7:		81	75.238	72.032	

8:		80	74.604	72.068	

9:		80	74.111	70.940	

10:		80	73.124	69.179	

11:		81	77.316	72.385	

12:		81	76.682	72.702	

13:		81	78.514	74.921	

14:		81	77.527	75.167	

15:		82	76.893	73.864	

16:		81	76.964	73.441	

17:		81	76.541	73.160	

18:		80	75.343	72.068	

19:		81	75.273	72.138	

%% Cell type:code id: tags:

``` python

tweet = u'RT: @MikeyMike I went to the #KanyeWest concert too!'

doc = nlp(tweet)

map(lambda x: (x.orth_, x.pos_, x.tag_, x.dep_), doc)

```

%% Output

[(u'RT', u'X', u'~', u'ROOT'),

 (u':', u'PUNCT', u',', u'ROOT'),

 (u'@MikeyMike', u'X', u'@', u'ROOT'),

 (u'I', u'PRON', u'O', u'_'),

 (u'went', u'VERB', u'V', u'ROOT'),

 (u'to', u'ADP', u'P', u'_'),

 (u'the', u'DET', u'D', u'MWE'),

 (u'#', u'PUNCT', u',', u'ROOT'),

 (u'KanyeWest', u'PROPN', u'^', u'ROOT'),

 (u'concert', u'NOUN', u'N', u'ROOT'),

 (u'too', u'ADV', u'R', u'_'),

 (u'!', u'PUNCT', u',', u'ROOT')]

%% Cell type:code id: tags:

``` python

```

Due to ongoing efforts to ensure you have the best possible learning experience, we may update the working files for this course from time to time.

We recommend that you download the latest version of these files from:

http://www.infiniteskills.com/02477

 No newline at end of file