In [3]:

    

%pip install --upgrade --quiet apache-beam[gcp]


    

    




Note: you may need to restart the kernel to use updated packages.

In [1]:

    

!rm -rf output.txt* beam-temp*


    

In [119]:

    

import apache_beam as beam
from apache_beam.ml.gcp import naturallanguageml as nlp

def parse_nlp_result(response):
    """
Pulls required info from a response that looks like this:

sentences {
  text {
    content: "I love walking along the Seine."
  }
  sentiment {
    magnitude: 0.699999988079071
    score: 0.699999988079071
  }
}
entities {
  name: "Seine"
  type: LOCATION
  metadata {
    key: "mid"
    value: "/m/0f3vz"
  }
  metadata {
    key: "wikipedia_url"
    value: "https://en.wikipedia.org/wiki/Seine"
  }
  salience: 1.0
  mentions {
    text {
      content: "Seine"
      begin_offset: 25
    }
    type: PROPER
  }
}
document_sentiment {
  magnitude: 0.699999988079071
  score: 0.699999988079071
}
language: "en"
    """
    def get_entity_value(entities, search_key):
        for entity in entities:
            return (entity.metadata[search_key])
        return ''
    
    return [
        # response, # entire string
        response.sentences[0].text.content, # first sentence
        [entity.name for entity in response.entities], # all entities
        [entity.metadata['wikipedia_url'] for entity in response.entities], # urls
        response.language,
        response.document_sentiment.score
    ]


features = nlp.types.AnnotateTextRequest.Features(
    extract_entities=True,
    extract_document_sentiment=True,
    extract_syntax=False
)

p = beam.Pipeline()
(p 
 | beam.Create(['Has President Obama been to Paris?', 'Sophie loves walking along the Seine.', "C'est terrible"])
 | beam.Map(lambda x : nlp.Document(x, type='PLAIN_TEXT'))
 | nlp.AnnotateText(features)
 | beam.Map(parse_nlp_result)
 | beam.io.WriteToText('output.txt')
)
result = p.run()
result.wait_until_finish()


    

    
Out[119]:





'DONE'

In [120]:

    

!cat output.txt*


    

    




['Has President Obama been to Paris?', ['Obama', 'Paris'], ['https://en.wikipedia.org/wiki/Barack_Obama', 'https://en.wikipedia.org/wiki/Paris'], 'en', 0.0]
["C'est terrible", [], [], 'fr', -0.8999999761581421]
['Sophie loves walking along the Seine.', ['Sophie', 'Seine'], ['', 'https://en.wikipedia.org/wiki/Seine'], 'en', 0.800000011920929]

In [17]:

    

%%bigquery
SELECT text FROM `bigquery-public-data.hacker_news.comments`
WHERE author = 'AF' LIMIT 10


    

    
Out[17]:







  

    

      

      
text
    
  
  

    

      
0
      
I think there's a major problem with this, and...
    
    

      
1
      
Speaking of Rails, there are other options in ...
    
    

      
2
      
I don't see the point in this as a serious pro...
    
    

      
3
      
Nope. It is a nice package, but there's too ma...
    
    

      
4
      
I'll be perfectly honest: what is popular on D...
    
    

      
5
      
Also keep in mind that Python has much better ...
    
    

      
6
      
I'm just wondering, what specifically about th...
    
    

      
7
      
Just get a basic knowledge of each of them.<p>...
    
    

      
8
      
Haven't we already discussed this? Google is n...
    
    

      
9
      
The general idea is interesting and possibly e...
    
  

In [74]:

    

%%writefile nlp_pipeline.py

PROJECT='ai-analytics-solutions'
BUCKET='ai-analytics-solutions-kfpdemo'
REGION='us-central1'

from datetime import datetime
import apache_beam as beam

def parse_nlp_result(response):
    return [
        # response, # entire string
        response.sentences[0].text.content,
        response.language,
        response.document_sentiment.score
    ]

def run():
    from apache_beam.ml.gcp import naturallanguageml as nlp
    
    features = nlp.types.AnnotateTextRequest.Features(
        extract_entities=True,
        extract_document_sentiment=True,
        extract_syntax=False
    )
    options = beam.options.pipeline_options.PipelineOptions()
    google_cloud_options = options.view_as(beam.options.pipeline_options.GoogleCloudOptions)
    google_cloud_options.project = PROJECT
    google_cloud_options.region = REGION
    google_cloud_options.job_name = 'nlpapi-{}'.format(datetime.now().strftime("%Y%m%d-%H%M%S"))
    google_cloud_options.staging_location = 'gs://{}/staging'.format(BUCKET)
    google_cloud_options.temp_location = 'gs://{}/temp'.format(BUCKET)
    options.view_as(beam.options.pipeline_options.StandardOptions).runner = 'DataflowRunner' # 'DirectRunner'

    p = beam.Pipeline(options=options)
    (p 
     | 'bigquery' >> beam.io.Read(beam.io.BigQuerySource(
         query="SELECT text FROM `bigquery-public-data.hacker_news.comments` WHERE author = 'AF' AND LENGTH(text) > 10",
         use_standard_sql=True))
      | 'txt'      >> beam.Map(lambda x : x['text'])
      | 'doc'      >> beam.Map(lambda x : nlp.Document(x, type='PLAIN_TEXT'))
    #  | 'todict'   >> beam.Map(lambda x : nlp.Document.to_dict(x))
      | 'nlp'      >> nlp.AnnotateText(features, timeout=10)
      | 'parse'    >> beam.Map(parse_nlp_result)
      | 'gcs'      >> beam.io.WriteToText('gs://{}/output.txt'.format(BUCKET), num_shards=1)
    )
    result = p.run()
    result.wait_until_finish()

if __name__ == '__main__':
    run()


    

    




Overwriting nlp_pipeline.py

In [75]:

    

!python3 nlp_pipeline.py


    

In [76]:

    

!gsutil cat gs://$BUCKET/output.txt*


    

    




["I think there's a major problem with this, and it is that discussions come in all shapes and sizes across the web.<p>Think about it.", 'en', -0.20000000298023224]
['It is just a joke that Facebook could be valued at $6 billion.', 'en', -0.5]
["This article doesn't make too much sense to me.<p>First of all, as Sam mentioned, the companies aren't that much different in size.", 'en', -0.4000000059604645]
['&#62; If only a real Lisp would come along that had the leadership, library, community and documentation of Python.<p>Someone was asking for that just the other day.', 'en', 0.0]
["If the original poster has his service out when he does this, it could end up being both very bad PR for her and very good PR for him.<p>In fact, her ripping off his idea could end up being better publicity than he would've gotten anywhere else.", 'en', -0.699999988079071]
["I've had my share of bad experiences with Apple products/computers.", 'en', 0.0]
['Of course.', 'en', 0.0]
["I don't think so.", 'en', -0.10000000149011612]
['This is just a single example.', 'en', 0.0]
["The general idea is interesting and possibly even right, but bloggers tend to overvalue and exaggerate blogging's reach.", 'en', 0.4000000059604645]
["You know I really do like apt, but it isn't perfect, and 'rough around the edges' isn't a bad description for it (I know he meant Ubuntu as a whole).<p>I've had multiple times that by using apt-get autoremove, my entire Ubuntu installation has been fubared.", 'en', 0.0]
["I don't mean this to come out as negatively as it might sound, but JRuby being faster than Ruby 1.8 isn't saying much; MRI is pretty slow.", 'en', -0.699999988079071]
['"Yes, but what you forget Paul backspace backspace backspace, Paul, is that Microsoft Vista has voice recognition.', 'en', 0.0]
["It is definitely something to consider, and while I don't think translating Python to Common Lisp would be all that difficult, I do think that getting the C libraries that Python wraps working with CL might be a significant effort.", 'en', 0.4000000059604645]
["You are right, it would've been a poor argument if that's all they said.", 'en', -0.10000000149011612]
['Seems pretty silly.', 'en', -0.5]
["I think the idea behind a 'web OS' is horrible.", 'en', -0.20000000298023224]
["I'm not a huge fan of it, but my immediate thought was 'emacs'.", 'en', -0.10000000149011612]
['The article is correct.', 'en', 0.0]
['There are advantages to both approaches.', 'en', -0.10000000149011612]
['Speaking of which, have you guys seen what Facebook has turned into with the advent of the platform?', 'en', -0.30000001192092896]
["Don't be so sure either of those are that important.<p>You can get a laptop with comparable power to some of the more expensive desktops now for about the same price.", 'en', -0.5]
['Is he serious about this being the coolest application to date?', 'en', 0.0]
['You realize what happened after the boom of the 1920s, right?', 'en', -0.30000001192092896]
['I thought I made it clear.', 'en', 0.30000001192092896]
["So basically Digg does what everyone else does?<p>I've always wondered if Digg was written using Python or another language instead of PHP, if they could squeeze better performance out of it.", 'en', -0.6000000238418579]
['&#62; It is increasingly dangerous to have a child past 30.', 'en', -0.30000001192092896]
["Well, if you wanted to ship a desktop application or such, you'd have to open source your code.", 'en', -0.4000000059604645]
["Kind of an old post, things have changed for both Rails/Django since then.<p>Though I'd like to comment on part of his article.", 'en', 0.0]
['Smalltalk is a really excellent language.', 'en', 0.10000000149011612]
['It is the internet.', 'en', 0.20000000298023224]
['That clip struck me as Kevin being really shady.', 'en', -0.6000000238418579]
['Is it?', 'en', 0.10000000149011612]
['Please be smart and go with the ThinkPad (parcticularly a T61p).<p>Right now they have a 25% off sale on them, not to mention you can get another 5% off if you have a Visa card, or are a student, or a number of other things.', 'en', 0.0]
['Speaking of Rails, there are other options in the Python world besides Django.<p>Pylons is a very Rails-y framework with the difference being that it is made to be easy to customize.', 'en', 0.0]
["I don't see a problem with it.", 'en', 0.10000000149011612]
['What is wrong with just voting up stories you like?', 'en', -0.4000000059604645]
["You are right about GEdit's power.", 'en', 0.5]
["I just wanted to say that if you ever plan on running Linux on your machine, don't get a Mac.<p>Ubuntu in particular works, but there's a lot of hacking to get it to do so and little issues abound (at least that's my experience).", 'en', -0.20000000298023224]
['Money is overrated.', 'en', -0.5]
['Nope.', 'en', 0.20000000298023224]
["Ubuntu really isn't difficult to setup at all on most PCs.", 'en', 0.5]
['What is the best alternative to an RDMS in most cases?', 'en', 0.800000011920929]
["I'll be perfectly honest: what is popular on Digg is in no way something to be proud of.<p>Digg is a complete waste of time.", 'en', -0.6000000238418579]
['What kind of apps do you mean by this?', 'en', -0.30000001192092896]
["Just get a basic knowledge of each of them.<p>CL is pretty special and has features you just won't find in most other languages (macros, interactive error-handling, generic functions), and raw speed (SBCL).<p>OCaml...I hear discussion about it, but personally having evaluated it, I don't know what kind of a future it has (I really doubt it will ever be a 'big' thing).", 'en', -0.10000000149011612]
['Hmm.', 'en', 0.0]
["Please tell me what is wrong with my statement.<p>I don't think it of respectful users, I think it of Digg users, which have never been shown to be intelligent or respectful of either the web site they use or anyone else for that matter.", 'en', -0.699999988079071]
["Heh, well if you are building YouTube, Dreamhost isn't going to do much good either. :)", 'en', 0.699999988079071]
['How are people monetizing Facebook aps?', 'en', -0.30000001192092896]
['One problem with this: people expect to do that in video games.', 'en', 0.10000000149011612]
["I don't see the equivalence.", 'en', -0.20000000298023224]
['Sorry...double post.', 'en', -0.10000000149011612]
["I hold the same disdain for walled networks you do, but ultimately I use Facebook and others do as well simply because it is convenient and easy to use.<p>I also wouldn't base my main application on it, but as a developer I don't mind it for smaller apps.", 'en', -0.4000000059604645]
["I've thought of this before, but the issue is, what if people don't pay up?", 'en', -0.10000000149011612]
["Yeah he definitely didn't start Reddit.", 'en', -0.30000001192092896]
["I'm just wondering, what specifically about the Mac UI gives you significant productivity boosts?", 'en', 0.699999988079071]
['The issue is EFI.', 'en', -0.10000000149011612]
["Haven't we already discussed this?", 'en', -0.20000000298023224]
['When did he ever imply that?', 'en', -0.30000001192092896]
['Wow.', 'en', -0.10000000149011612]
['I am reminded of similar hyping and grandiose claims that Aaron made when he announced both Infogami and web.py.', 'en', -0.5]
["I don't see the point in this as a serious production language.", 'en', 0.0]
['Can someone explain to me why all these startup companies end up in California?', 'en', 0.20000000298023224]
['The GPL licensing would be an issue for some.<p>Edit: Unfortunately Logix seems to have been dead since 2005.<p><a href="http://common-lisp.net/project/python-on-lisp/" rel="nofollow">http://common-lisp.net/project/python-on-lisp/</a>', 'en', 0.0]
["I'm sure you know that the idea of using code in templates is controversial anyway.", 'en', 0.5]
["Unless you've really got a Ruby fetish, there's just not much of a reason to use it.<p>If you want a similar language with similar productivity, Python gives you that and speed, Unicode support at the language level, and lots of libraries.<p>And in web frameworks alone you've got Django, Pylons, Turbogears, web.py, etc.", 'en', 0.0]
["You can directly compare the speed of an application or a language.<p>It is not subjective to say 'C is faster than Perl' or 'Perl is faster than Ruby'.", 'en', 0.0]
['Paul just how much impact do you think that Reddit being initially implemented in Lisp had?<p>I mean, sure, relative numbers of Lisp users might have been pretty small, but the very fact that it was written in Lisp may have gotten fans and detractors to at least hear about the site (probably the hardest thing to do), and then once they checked it out start using it.<p>So how important do you think that was to initial popularity?', 'en', -0.20000000298023224]
['How many books do you need for a web framework?<p>Django is pretty simple.', 'en', -0.20000000298023224]
["That isn't always an option (truth be told it isn't an option many people have).", 'en', -0.10000000149011612]
["I've recognized this 'mental transaction cost' for awhile, but never been able to put it into words.", 'en', 0.20000000298023224]
["I don't mean to be rude about it, but groopvine.com is a terrible domain name, imo.", 'en', -0.4000000059604645]
['People continue to bring this post up.', 'en', -0.10000000149011612]
["I'm fairly certain it isn't revolutionary, it just shows the rampant immaturity of Digg's user base and their mob justice that has no respect and knows no bounds.", 'en', -0.6000000238418579]
["I've used both.", 'en', 0.0]
['Never.', 'en', -0.4000000059604645]
["Google isn't an ad-supported business, though.", 'en', -0.20000000298023224]
['Hey guys...<p>I own an Apple and have been trying to get Ubuntu running on it.', 'en', -0.10000000149011612]
['Those are called blogs, no?', 'en', -0.10000000149011612]
['You are right about the runtimes.', 'en', -0.10000000149011612]
['Hey gwen...what do you mean by this: "If JS is Lisp in disguise"?', 'en', 0.0]
["As I mentioned in another post, I'm aware of the qualities of SBCL.", 'en', -0.20000000298023224]
['Also keep in mind that Python has much better (quantity and quality) libraries.<p>Compare, for example, SQLAlchemy to ActiveRecord.', 'en', -0.20000000298023224]
['This is why if you have an idea you might as well do it.', 'en', -0.4000000059604645]
['It is unfortunate because it seems that those that succeed (especially in the computer industry) are the biggest jerks.<p>The article does mention Gates, who was involved in some pretty shady stuff when it comes to establishing Windows.', 'en', -0.4000000059604645]
['Because projectors require a few things that TVs do not.<p>1.', 'en', -0.4000000059604645]