Initial commit

{\rtf1\ansi\ansicpg1252\cocoartf1038\cocoasubrtf350

{\fonttbl\f0\fswiss\fcharset0 Helvetica;}

{\colortbl;\red255\green255\blue255;}

\margl1440\margr1440\vieww9000\viewh8400\viewkind0

\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural\pardirnatural

\f0\fs24 \cf0 No work files for this lesson...}

 No newline at end of file

%% Cell type:markdown id: tags:

### Dependencies

* pip install spacy, pandas, matplotlib, pathlib

* python -m spacy.en.download

%% Cell type:code id: tags:

``` python

#import sys

#sys.path.append('/home/jupyter/site-packages/')

from IPython.display import SVG, display

import spacy

import pandas as pd

import matplotlib.pyplot as plt

from datetime import datetime

%matplotlib inline

```

%% Cell type:code id: tags:

``` python

nlp = spacy.load('en')

```

%% Cell type:markdown id: tags:

### Accessing Tokens and Spans

%% Cell type:code id: tags:

``` python

import pandas as pd

def info(obj):

    return {'type':type(obj),'__str__': str(obj)}

text = u"""spaCy excels at large-scale information extraction tasks. 

It's written from the ground up in carefully memory-managed Cython. """

document = nlp(text)

token = document[0]

span = document[0:3]

pd.DataFrame(map(info, [token,span,document]))

```

%% Output

                                             __str__  \

0                                              spaCy   

1                                    spaCy excels at   

2  spaCy excels at large-scale information extrac...   

                                type  

0  <type 'spacy.tokens.token.Token'>  

1    <type 'spacy.tokens.span.Span'>  

2      <type 'spacy.tokens.doc.Doc'>  

%% Cell type:markdown id: tags:

### Sentence boundary detection

%% Cell type:code id: tags:

``` python

print document.sents

for sent in document.sents:

    print sent

```

%% Output

<generator object at 0x7efbd365a738>

spaCy excels at large-scale information extraction tasks. 

It's written from the ground up in carefully memory-managed Cython.

%% Cell type:markdown id: tags:

### Tokenization

%% Cell type:code id: tags:

``` python

for token in document:

    print token

```

%% Output

spaCy

excels

at

large

-

scale

information

extraction

tasks

.

It

's

written

from

the

ground

up

in

carefully

memory

-

managed

Cython

.

%% Cell type:code id: tags:

``` python

### Morphological decomposition

```

%% Cell type:code id: tags:

``` python

token = document[13]

print "text: %s" % token.text

print "suffix: %s" % token.suffix_ 

print "lemma: %s" % token.lemma_

```

%% Cell type:code id: tags:

``` python

### Part of Speech Tagging

```

%% Cell type:code id: tags:

``` python

#Part of speech and Dependency tagging

attrs = map(lambda token: {

                     "token":token

                   , "part of speech":token.pos_

                   , "Dependency" : token.dep_}

                    , document)

pd.DataFrame(attrs)

```

%% Cell type:code id: tags:

``` python

### Noun Chunking

```

%% Cell type:code id: tags:

``` python

print "noun chunks: {}".format(list(document.noun_chunks))

```

%% Cell type:code id: tags:

``` python

### Named Entity Recognition

```

%% Cell type:code id: tags:

``` python

ents = [(ent, ent.root.ent_type_) for ent in document.ents]

print "entities: {}".format(ents)

```

%% Cell type:code id: tags:

``` python

### Text Similarity (Using Word Vectors)

```

%% Cell type:code id: tags:

``` python

#document, span, and token similarity

def plot_similarities(similarities, target):

    import matplotlib.pyplot as plt

    %matplotlib inline

    f, ax = plt.subplots(1)

    index = range(len(similarities))

    ax.barh(index, similarities)

    ax.set_yticks(index)

    ax.set_yticklabels(document2)

    ax.grid(axis='x')

    ax.set_title("Similarity to '{}'".format(target))

    plt.show()

computer = nlp(u'computer')

document2 = nlp(u'You might be using a machine running Windows')

similarities = map(lambda token: token.similarity(computer),document2)

plot_similarities(similarities, computer)

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:markdown id: tags:

### Dependencies for this notebook:

<br>

* pip install spacy, pandas, matplotlib

* python -m spacy.en.download

%% Cell type:code id: tags:

``` python

from IPython.display import SVG, display

import spacy

import pandas as pd

import matplotlib.pyplot as plt

from datetime import datetime

%matplotlib inline

```

%% Cell type:code id: tags:

``` python

#encode some text as uncode

text = u"I'm executing this code on an Apple Computer."

#instantiate a language model

#to download language model: python -m spacy.en.download 

nlp = spacy.load('en') # or spacy.en.English()

#create a document

document = nlp(text)

```

%% Cell type:code id: tags:

``` python

for function in nlp.pipeline:

    print function

```

%% Cell type:code id: tags:

``` python

### Modifying the Language Model

def identify_starwars(doc):

    for token in doc:

        if token.text == u'starwars':

            token.tag_ = u'NNP'

def return_pipeline(nlp):

    return [nlp.tagger, nlp.parser, nlp.matcher, nlp.entity, identify_starwars]

text = u"I loved all of the starwars movies"

custom_nlp = spacy.load('en', create_pipeline=return_pipeline)

new_document = custom_nlp(text)

for function in custom_nlp.pipeline:

    print function

```

%% Cell type:code id: tags:

``` python

texts = [u'You have brains in your head.'] * 10000

for doc in nlp.pipe(texts,n_threads=4):

    doc.is_parsed

```

%% Cell type:code id: tags:

``` python

### Deploying Model on Many Texts with .pipe

runtimes = {}

for thread_count in [1,2,3,4,8]:

    t0 =  datetime.now() 

    #Create generator of processed documents

    processed_documents = nlp.pipe(texts,n_threads=thread_count)

    #Iterate over generator

    for doc in processed_documents: 

        #pipeline is only run once we access the generator

        doc.is_parsed 

    t1 = datetime.now()

    runtimes[thread_count] = (t1 - t0).total_seconds()

ax = pd.Series(runtimes).plot(kind = 'bar')

ax.set_ylabel("Runtime (Seconds) with N Threads")

plt.show()

```

%% Cell type:markdown id: tags:

### Accessing Tokens and Spans

%% Cell type:code id: tags:

``` python

import pandas as pd

def info(obj):

    return {'type':type(obj),'__str__': str(obj)}

text = u"""spaCy excels at large-scale information extraction tasks. 

It's written from the ground up in carefully memory-managed Cython. """

document = nlp(text)

token = document[0]

span = document[0:3]

pd.DataFrame(map(info, [token,span,document]))

```

%% Cell type:markdown id: tags:

### Sentence boundary detection

%% Cell type:code id: tags:

``` python

print document.sents

print

for sent in document.sents:

    print sent

```

%% Cell type:markdown id: tags:

### Tokenization

%% Cell type:code id: tags:

``` python

for token in document:

    print token

```

%% Cell type:markdown id: tags:

### Morphological decomposition

%% Cell type:code id: tags:

``` python

token = document[13]

print "text: %s" % token.text

print "suffix: %s" % token.suffix_ 

print "lemma: %s" % token.lemma_

```

%% Cell type:markdown id: tags:

### Part of Speech Tagging

%% Cell type:code id: tags:

``` python

#Part of speech and Dependency tagging

attrs = map(lambda token: {

                     "token":token

                   , "part of speech":token.pos_

                   , "Dependency" : token.dep_}

                    , document)

pd.DataFrame(attrs)

```

%% Cell type:markdown id: tags:

### Noun Chunking

%% Cell type:code id: tags:

``` python

print "noun chunks: {}".format(list(document.noun_chunks))

```

%% Cell type:markdown id: tags:

### Named Entity Recognition

%% Cell type:code id: tags:

``` python

ents = [(ent, ent.root.ent_type_) for ent in document.ents]

print "entities: {}".format(ents)

```

%% Cell type:markdown id: tags:

### Text Similarity (Using Word Vectors)

%% Cell type:code id: tags:

``` python

#document, span, and token similarity

def plot_similarities(similarities, target):

    import matplotlib.pyplot as plt

    %matplotlib inline

    f, ax = plt.subplots(1)

    index = range(len(similarities))

    ax.barh(index, similarities)

    ax.set_yticks([i + 0. for i in index])

    ax.set_yticklabels(document2)

    ax.grid(axis='x')

    ax.set_title("Similarity to '{}'".format(target))

    plt.show()

    return ax

computer = nlp(u'computer')

document2 = nlp(u'You might be using a machine running Windows')

similarities = map(lambda token: token.similarity(computer),document2)

ax = plot_similarities(similarities, computer)

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:code id: tags:

``` python

```

%% Cell type:code id: tags:

``` python

```