notebook.community

Edit and run 





In [1]:



    


import index



    











In [2]:



    


import sys



    











In [3]:



    


sys.path.append("/Users/karve/code/quilt/lambdas/es/indexer/test")



    











In [4]:



    


RECORDS = {
    "ObjectRemoved:Delete": {
        "eventVersion": "2.1",
        "eventSource": "aws:s3",
        "awsRegion": "us-east-1",
        "eventTime": "2020-05-22T00:32:20.515Z",
        "eventName": "ObjectRemoved:Delete",
        "userIdentity": {
            "principalId": "EXAMPLE"
        },
        "requestParameters": {
            "sourceIPAddress": "127.0.0.1"
        },
        "responseElements": {
            "x-amz-request-id": "EXAMPLE123456789",
            "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
        },
        "s3": {
            "s3SchemaVersion": "1.0",
            "configurationId": "testConfigRule",
            "bucket": {
                "name": "test-bucket",
                "ownerIdentity": {
                    "principalId": "EXAMPLE"
                },
                "arn": "arn:aws:s3:::test-bucket"
            },
            "object": {
                "key": "hello+world.txt",
                "sequencer": "0A1B2C3D4E5F678901"
            }
        }
    },
    "ObjectCreated:Put": {
        "eventVersion": "2.1",
        "eventSource": "aws:s3",
        "awsRegion": "us-east-1",
        "eventTime": "2020-05-22T00:32:20.515Z",
        "eventName": "ObjectCreated:Put",
        "userIdentity": {
            "principalId": "EXAMPLE"
        },
        "requestParameters": {
            "sourceIPAddress": "127.0.0.1"
        },
        "responseElements": {
            "x-amz-request-id": "EXAMPLE123456789",
            "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
        },
        "s3": {
            "s3SchemaVersion": "1.0",
            "configurationId": "testConfigRule",
            "bucket": {
                "name": "test-bucket",
                "ownerIdentity": {
                    "principalId": "EXAMPLE"
                },
                "arn": "arn:aws:s3:::test-bucket"
            },
            "object": {
                "key": "hello+world.txt",
                "size": 100,
                "eTag": "123456",
                "sequencer": "0A1B2C3D4E5F678901"
            }
        }
    },
    # This template is based on an actual S3 event on 5-21-2020, no known docs
    # on this particular event type
    "ObjectRemoved:DeleteMarkerCreated": {
        "awsRegion": "us-east-1",
        "eventName": "ObjectRemoved:DeleteMarkerCreated",
        "eventSource": "aws:s3",
        "eventTime": "2020-05-22T00:32:20.515Z",
        "eventVersion": "2.1",
        "requestParameters": {
            "sourceIPAddress": "127.0.0.1"
        },
        "responseElements": {
            "x-amz-request-id": "EXAMPLE123456789",
            "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
        },
        "s3": {
            "bucket": {
                "name": "test-bucket",
                "ownerIdentity": {
                    "principalId": "EXAMPLE"
                },
                "arn": "arn:aws:s3:::test-bucket"
            },
            "configurationId": "testConfigRule",
            "object": {
                "key": "hello+world.txt",
                "eTag": "123456",
                "sequencer": "0A1B2C3D4E5F678901",
                "versionId": "1313131313131.Vier50HdNbi7ZirO65"
            },
            "s3SchemaVersion": "1.0"
        },
        "userIdentity": {
            "principalId": "EXAMPLE"
        }
    }
}



    











In [5]:



    


dk = RECORDS["ObjectRemoved:Delete"].keys()
pk = RECORDS["ObjectCreated:Put"].keys()
mk = RECORDS["ObjectRemoved:DeleteMarkerCreated"].keys()



    











In [6]:



    


od = RECORDS["ObjectRemoved:Delete"]
op = RECORDS["ObjectCreated:Put"]
odm = RECORDS["ObjectRemoved:DeleteMarkerCreated"]



    











In [7]:



    


od['s3']



    


















    
Out[7]:








{'s3SchemaVersion': '1.0',
 'configurationId': 'testConfigRule',
 'bucket': {'name': 'test-bucket',
  'ownerIdentity': {'principalId': 'EXAMPLE'},
  'arn': 'arn:aws:s3:::test-bucket'},
 'object': {'key': 'hello+world.txt', 'sequencer': '0A1B2C3D4E5F678901'}}

















In [8]:



    


op["s3"]
# add size and etag to s3



    


















    
Out[8]:








{'s3SchemaVersion': '1.0',
 'configurationId': 'testConfigRule',
 'bucket': {'name': 'test-bucket',
  'ownerIdentity': {'principalId': 'EXAMPLE'},
  'arn': 'arn:aws:s3:::test-bucket'},
 'object': {'key': 'hello+world.txt',
  'size': 100,
  'eTag': '123456',
  'sequencer': '0A1B2C3D4E5F678901'}}

















In [9]:



    


odm["s3"]
# add size, versionId to s3



    


















    
Out[9]:








{'bucket': {'name': 'test-bucket',
  'ownerIdentity': {'principalId': 'EXAMPLE'},
  'arn': 'arn:aws:s3:::test-bucket'},
 'configurationId': 'testConfigRule',
 'object': {'key': 'hello+world.txt',
  'eTag': '123456',
  'sequencer': '0A1B2C3D4E5F678901',
  'versionId': '1313131313131.Vier50HdNbi7ZirO65'},
 's3SchemaVersion': '1.0'}

















In [10]:



    


from copy import deepcopy



    











In [11]:



    


EVENT_CORE = deepcopy(op)



    











In [12]:



    


for d in ("eTag", "size", "versionId"):
    if d in EVENT_CORE["s3"]["object"]:
        del EVENT_CORE["s3"]["object"][d]



    











In [13]:



    


EVENT_CORE



    


















    
Out[13]:








{'eventVersion': '2.1',
 'eventSource': 'aws:s3',
 'awsRegion': 'us-east-1',
 'eventTime': '2020-05-22T00:32:20.515Z',
 'eventName': 'ObjectCreated:Put',
 'userIdentity': {'principalId': 'EXAMPLE'},
 'requestParameters': {'sourceIPAddress': '127.0.0.1'},
 'responseElements': {'x-amz-request-id': 'EXAMPLE123456789',
  'x-amz-id-2': 'EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH'},
 's3': {'s3SchemaVersion': '1.0',
  'configurationId': 'testConfigRule',
  'bucket': {'name': 'test-bucket',
   'ownerIdentity': {'principalId': 'EXAMPLE'},
   'arn': 'arn:aws:s3:::test-bucket'},
  'object': {'key': 'hello+world.txt', 'sequencer': '0A1B2C3D4E5F678901'}}}

















In [31]:



    


import pprint



    











In [38]:



    


pprint.pprint(EVENT_CORE, width=100)



    


















    






{'awsRegion': 'us-east-1',
 'eventName': 'ObjectCreated:Put',
 'eventSource': 'aws:s3',
 'eventTime': '2020-05-22T00:32:20.515Z',
 'eventVersion': '2.1',
 'requestParameters': {'sourceIPAddress': '127.0.0.1'},
 'responseElements': {'x-amz-id-2': 'EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH',
                      'x-amz-request-id': 'EXAMPLE123456789'},
 's3': {'bucket': {'arn': 'arn:aws:s3:::test-bucket',
                   'name': 'test-bucket',
                   'ownerIdentity': {'principalId': 'EXAMPLE'}},
        'configurationId': 'testConfigRule',
        'object': {'key': 'hello+world.txt', 'sequencer': '0A1B2C3D4E5F678901'},
        's3SchemaVersion': '1.0'},
 'userIdentity': {'principalId': 'EXAMPLE'}}

















In [14]:



    


def make_event(name, eTag="", key="",  size=0, versionId=""):
    e = deepcopy(EVENT_CORE)
    e["eventName"] = name

    if key:
        e["s3"]["object"]["key"] = key
    if eTag:
        e["s3"]["object"]["eTag"] = eTag
    if size:
        e["s3"]["object"]["size"] = size
    if versionId:
        e["s3"]["object"]["versionId"] = versionId
    
    return e



    











In [15]:



    


make_event("ObjectRemoved:Delete") == od



    


















    
Out[15]:








True

















In [25]:



    


make_event(
    "ObjectRemoved:DeleteMarkerCreated",
    versionId="1313131313131.Vier50HdNbi7ZirO65",
    eTag="123456",
    key="hello+world.txt"
) == odm



    


















    
Out[25]:








True

















In [30]:



    


make_event(
    "ObjectCreated:Put",
    eTag="123456",
    key="hello+world.txt",
    size=100
) == op



    


















    
Out[30]:








True

















In [2]:



    


def outer(i=1):
    def inner():
        print(i)
    
    inner()



    











In [6]:



    


outer(19)



    


















    






19

















In [7]:



    


mraw =  [{'index': {'_index': 'test-bucket', '_id': 'hello world.txt:1313131313131.Vier50HdNbi7ZirO65'}}, {'comment': '', 'content': 'Hello World!', 'etag': '123456', 'event': 'ObjectCreated:Put', 'ext': '.txt', 'key': 'hello world.txt', 'last_modified': '2020-05-30T02:36:45.186960+00:00', 'meta_text': ' ', 'size': 100, 'target': '', 'updated': '2020-05-30T02:36:45.190205', 'version_id': '1313131313131.Vier50HdNbi7ZirO65'}]



    











In [8]:



    


mraw



    


















    
Out[8]:








[{'index': {'_index': 'test-bucket',
   '_id': 'hello world.txt:1313131313131.Vier50HdNbi7ZirO65'}},
 {'comment': '',
  'content': 'Hello World!',
  'etag': '123456',
  'event': 'ObjectCreated:Put',
  'ext': '.txt',
  'key': 'hello world.txt',
  'last_modified': '2020-05-30T02:36:45.186960+00:00',
  'meta_text': ' ',
  'size': 100,
  'target': '',
  'updated': '2020-05-30T02:36:45.190205',
  'version_id': '1313131313131.Vier50HdNbi7ZirO65'}]

















In [9]:



    


mact = [{'index': {'_index': 'test-bucket', '_id': 'hello world.txt:1313131313131.Vier50HdNbi7ZirO65'}}]



    











In [10]:



    


mact



    


















    
Out[10]:








[{'index': {'_index': 'test-bucket',
   '_id': 'hello world.txt:1313131313131.Vier50HdNbi7ZirO65'}}]

















In [20]:



    


[next(iter(y)) for y,z in [x.items() for x in mact]]



    


















    






---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-20-04b9ec723216> in <module>
----> 1 [next(iter(y)) for y,z in [x.items() for x in mact]]

<ipython-input-20-04b9ec723216> in <listcomp>(.0)
----> 1 [next(iter(y)) for y,z in [x.items() for x in mact]]

ValueError: not enough values to unpack (expected 2, got 1)
















In [24]:



    


[next(iter(x.keys())) for x in mact]



    


















    
Out[24]:








['index']

















In [25]:



    


[{k:v} for k,v in [a.items() for a in mact]]



    


















    






---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-25-aba53cb14f57> in <module>
----> 1 [{k:v} for k,v in [a.items() for a in mact]]

<ipython-input-25-aba53cb14f57> in <listcomp>(.0)
----> 1 [{k:v} for k,v in [a.items() for a in mact]]

ValueError: not enough values to unpack (expected 2, got 1)
















In [33]:



    


[{k:v} for k,v in [x for x in mact]]



    


















    






---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-33-1ae2afd23890> in <module>
----> 1 [{k:v} for k,v in [x for x in mact]]

<ipython-input-33-1ae2afd23890> in <listcomp>(.0)
----> 1 [{k:v} for k,v in [x for x in mact]]

ValueError: not enough values to unpack (expected 2, got 1)
















In [41]:



    


[k,v in x.items() for x in mact]



    


















    






  File "<ipython-input-41-80964e041aa5>", line 1
    [k,v in x.items() for x in mact]
                        ^
SyntaxError: invalid syntax

















In [ ]:

Content source: quiltdata/quilt

Similar notebooks:

notebook.community | gallery | about