Code From Book


Code from Book

2. Getting Started


In [ ]:
! whoami

In [ ]:
%%bash 
echo $0

In [ ]:
! echo 'hello'

In [ ]:
import os
os.__file__

In [ ]:
%timeit 9999 in range(10000)

In [ ]:
%%javascript
var d = 10
alert("d = " + d)

3. Thinking in Binary


In [ ]:
! file my_image.jpg

In [ ]:
import magic
print magic.from_file("my_image.jpg")

In [ ]:
if magic.from_file("upload.jpg", mime=True) == "image/jpeg":
    continue_uploading("upload.jpg")
else:
    alert("Sorry! This file type is not allowed")

In [ ]:
import imghdr
print imghdr.what("path/to/my/file.ext")

In [ ]:
import binascii

def spoof_file(file, magic_number):
    magic_number = binascii.unhexlify(magic_number)
    with open(file, "r+b") as f:
        old = f.read()
        f.seek(0)
        f.write(magic_number + old)

In [ ]:
! xxd -b my_file.docx | less

In [ ]:
! du -h my_file.docx

In [ ]:
def to_ascii_bytes(string):
    return " ".join(format(ord(char), '08b') for char in string)

In [ ]:
string = "my ascii string"
"".join(hex(ord(char))[2:] for char in string)

In [ ]:
hex_string = "6d7920617363696920737472696e67"
print hex_string.decode("hex")
print "".join(chr(int(hex_string[i:i+2], 16)) for i in range(0, len(hex_string), 2))

In [ ]:
# adapted from https://code.activestate.com/recipes/142812-hex-dumper/
def hexdump(string, length=8):
    result = []
    digits = 4 if isinstance(string, unicode) else 2

    for i in xrange(0, len(string), length):
        s = string[i:i + length]
        hexa = "".join("{:0{}X}".format(ord(x), digits) for x in s)
        text = "".join(x if 0x20 <= ord(x) < 0x7F else '.' for x in s)
        result.append("{:04X}   {:{}}   {}".format(i, hexa, length * (digits + 1), text))

    return '\n'.join(result)

In [ ]:
print hexdump("The quick brown fox jumps over the lazy dog")

In [ ]:
import struct

num = 0x103e4
struct.pack("I", 0x103e4)

In [ ]:
string = '\xe4\x03\x01\x00'
struct.unpack("i", string)

In [ ]:
bytes = '\x01\xc2'
struct.pack("<h", struct.unpack(">h", bytes)[0])

In [ ]:
import base64

base64.b64encode('encodings are fun...')

In [ ]:
print base64.b64decode(_)

In [ ]:
string = "hello\x00"
binary_string = ' '.join('{:08b}'.format(ord(char)) for char in string)
" ".join(binary_string[i:i+6] for i in range(0, len(binary_string), 6))

In [ ]:
bin_string = '011010 000110 010101 101100 011011 000110 111100 000000'
[int(b, 2) for b in bin_string.split()]

In [ ]:
%%bash
echo -n hello | base64
echo aGVsbG8= | base64 --decode && echo

In [ ]:
u'◑ \u2020'.encode('utf8')

In [ ]:
'\xe2\x97\x91 \xe2\x80\xa0'.decode('utf8')

In [ ]:
unicode('\xe2\x97\x91 \xe2\x80\xa0', encoding='utf8')

In [ ]:
utf8_string = 'Åêíòü'
utf8_string

In [ ]:
unicode_string = utf8_string.decode('utf8')
unicode_string

In [ ]:
unicode_string.encode('mac roman')

In [ ]:
'Åêíòü'.decode('utf8').encode('ascii') # Raises UnicodeEncodeError

In [ ]:
! chardetect uni.txt another_file.txt

In [ ]:
file = """潍楪慢敫椠⁳桴⁥慧扲敬⁤整瑸琠慨⁴獩琠敨爠獥汵⁴景琠硥⁴敢湩⁧敤潣敤⁤獵湩⁧湡甠楮瑮湥敤⁤档
牡捡整⁲湥潣楤杮楷桴挠浯汰瑥汥⁹湵敲慬整⁤湯獥景整牦浯愠搠晩敦敲瑮眠楲楴杮猠獹整⹭‧⠊慔敫
牦浯攠⹮楷楫数楤⹡牯⥧"""

print file.decode('utf8').encode('utf16')

In [ ]:
import ftfy
ftfy.fix_text(u"“Mojibake“ can be fixed.")

In [ ]:
x = 0b1111
y = 0b1010
bin(int("{:b}{:b}".format(x, y), 2))

In [ ]:
bin(x << 4 | y)

4. Cryptography


In [ ]:
import random
import string

r = random.SystemRandom()

# Get a random integer between 0 and 20
r.randint(0, 20)

# Get a random number between 0 and 1
r.random()

# Generate a random 40-bit number
r.getrandbits(40)

# Choose a random item from a string or list
chars = string.printable
r.choice(chars)

# Randomize the order of a sequence
seq = ['a', 'b', 'c', 'd', 'e']
r.shuffle(seq)

In [ ]:
"ALLIGATOR".encode('rot13')

In [ ]:
"NYYVTNGBE".encode('rot13')

In [ ]:
plaintext = "A secret-ish message!"
"".join(chr((ord(c) + 20) % 256) for c in plaintext)

In [ ]:
ciphertext = 'U4\x87yw\x86y\x88A}\x87|4\x81y\x87\x87u{y5'
"".join(chr((ord(c) - 20) % 256) for c in ciphertext)

In [ ]:
plaintext = 0b110100001101001
one_time_pad = 0b110000011100001
bin(plaintext ^ one_time_pad)

In [ ]:
decrypted = 0b100010001000 ^ one_time_pad
format(decrypted, 'x').decode('hex')

In [ ]:
import os
import binascii

# ASCII-encoded plaintext
plaintext = "this is a secret message"
plaintext_bits = int(binascii.hexlify(plaintext), 16)

print "plaintext (ascii):", plaintext
print "plaintext (hex):", plaintext_bits

# Generate the one-time pad
onetime_pad = int(binascii.hexlify(os.urandom(len(plaintext))), 16)

print "one-time pad: (hex):", onetime_pad

# Encrypt plaintext using XOR operation with one-time pad
ciphertext_bits = plaintext_bits ^ onetime_pad

print "encrypted text (hex):", ciphertext_bits

# Decrypt using XOR operation with one-time pad
decrypted_text = ciphertext_bits ^ onetime_pad
decrypted_text = binascii.unhexlify(hex(decrypted_text)[2:-1])

print "decrypted text (ascii):", decrypted_text

In [ ]:
import random
import binascii

p1 = "this is the part where you run away"
p2 = "from bad cryptography practices."

# pad plaintexts with spaces to ensure equal length
p1 = p1.ljust(len(p2))
p2 = p2.ljust(len(p1))
    
p1 = int(binascii.hexlify(p1), 16)
p2 = int(binascii.hexlify(p2), 16)

# get random one-time pad
otp = random.SystemRandom().getrandbits(p1.bit_length())

# encrypt
c1 = p1 ^ otp
c2 = p2 ^ otp  # otp reuse...not good!

print "c1 ^ c2 == p1 ^ p2 ?", c1 ^ c2 == p1 ^ p2
print "c1 ^ c2 =", hex(c1 ^ c2)

# the crib
crib = " the "
crib = int(binascii.hexlify(crib), 16)

xored = c1 ^ c2

print "crib =", hex(crib)

cbl = crib.bit_length()
xbl = xored.bit_length()

print
mask = (2**(cbl + 1) - 1)
fill = len(str(xbl / 8))

# crib dragging
for s in range(0, xbl - cbl + 8, 8):
    xor = (xored ^ (crib << s)) & (mask << s)
    out = binascii.unhexlify(hex(xor)[2:-1])
    
    print "{:>{}}   {}".format(s/8, fill, out)

In [ ]:
from cryptography.fernet import Fernet
key = Fernet.generate_key()
f = Fernet(key)
ciphertext = f.encrypt("this is my plaintext")
decrypted = f.decrypt(ciphertext)
print decrypted

In [ ]:
import os
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

pt = "my plaintext"

backend = default_backend()
key = os.urandom(32)
iv = os.urandom(16)

padder = padding.PKCS7(128).padder()
pt = padder.update(pt) + padder.finalize()

cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend)
encryptor = cipher.encryptor()
ct = encryptor.update(pt) + encryptor.finalize()
decryptor = cipher.decryptor()
out = decryptor.update(ct) + decryptor.finalize()

unpadder = padding.PKCS7(128).unpadder()
out = unpadder.update(out) + unpadder.finalize()
print out

In [ ]:
nonce = os.urandom(64/8)
nonce

In [ ]:
import hashlib
hashlib.md5("hash me please").hexdigest()

In [ ]:
! md5 -s 'hash me please'

In [ ]:
hashlib.sha1("hash me please").hexdigest()

In [ ]:
! echo 'hash me please' | openssl dgst -sha1

In [ ]:
m1 = binascii.unhexlify("d131dd02c5e6eec4693d9a0698aff95c2fcab58712467eab4004583eb8fb7f8955ad340609f4b30283e488832571415a085125e8f7cdc99fd91dbdf280373c5bd8823e3156348f5bae6dacd436c919c6dd53e2b487da03fd02396306d248cda0e99f33420f577ee8ce54b67080a80d1ec69821bcb6a8839396f9652b6ff72a70")

m2 = binascii.unhexlify("d131dd02c5e6eec4693d9a0698aff95c2fcab50712467eab4004583eb8fb7f8955ad340609f4b30283e4888325f1415a085125e8f7cdc99fd91dbd7280373c5bd8823e3156348f5bae6dacd436c919c6dd53e23487da03fd02396306d248cda0e99f33420f577ee8ce54b67080280d1ec69821bcb6a8839396f965ab6ff72a70")

hashlib.md5(m1).hexdigest() == hashlib.md5(m1).hexdigest()

In [ ]:
import os
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
from cryptography.hazmat.backends import default_backend

backend = default_backend()
salt = os.urandom(16)

kdf = Scrypt(salt=salt, length=64, n=2**14, r=8, p=1, backend=backend)
key = kdf.derive("your favorite password")
key

In [ ]:
kdf = Scrypt(salt=salt, length=64, n=2**14, r=8, p=1, backend=backend)
kdf.verify("your favorite password", key)

In [ ]:
import hmac
import hashlib

secret_key = "my secret key"
ciphertext = "my ciphertext"

# generate HMAC
h = hmac.new(key=secret_key, msg=ciphertext, digestmod=hashlib.sha256)
print h.hexdigest()

# verify HMAC
hmac.compare_digest(h.hexdigest(), h.hexdigest())

In [ ]:
p = 9576890767
q = 1299827
n = p * q
print n

In [ ]:
e = 65537
phi = (p - 1) * (q - 1)
phi % e != 0

In [ ]:
import sympy

d = sympy.numbers.igcdex(e, phi)[0]
print d

In [ ]:
m = 12345
c = pow(m, e, n)
print c

In [ ]:
pow(c, d, n)

In [ ]:
m = 0
while pow(m, e, n) != c:
    m += 1
print m

In [ ]:
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization

private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend())

public_key = private_key.public_key()

private_pem = private_key.private_bytes(encoding=serialization.Encoding.PEM,  
                                        format=serialization.PrivateFormat.PKCS8, 
                                        encryption_algorithm=serialization.BestAvailableEncryption('your password here'))

public_pem = public_key.public_bytes(encoding=serialization.Encoding.PEM,  
                                     format=serialization.PublicFormat.SubjectPublicKeyInfo)

print public_pem
print private_pem

In [ ]:
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
import base64

with open("path/to/public_key.pem", "rb") as key_file:
    public_key = serialization.load_pem_public_key(key_file.read(), 
                                                   backend=default_backend())

message = "your secret message"
ciphertext = public_key.encrypt(message, 
                                padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()), 
                                             algorithm=hashes.SHA256(), 
                                             label=None))
b64_ciphertext = base64.urlsafe_b64encode(ciphertext)
print b64_ciphertext

In [ ]:
plaintext = private_key.decrypt(ciphertext, 
                                padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()), 
                                             algorithm=hashes.SHA256(), 
                                             label=None))
print plaintext

In [ ]:
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding

signer = private_key.signer(padding.PSS(mgf=padding.MGF1(hashes.SHA256()), 
                                        salt_length=padding.PSS.MAX_LENGTH), 
                            hashes.SHA256())
message = "A message of arbitrary length"
signer.update(message)
signature = signer.finalize()
signature

In [ ]:
public_key = private_key.public_key()
verifier = public_key.verifier(signature, 
                               padding.PSS(mgf=padding.MGF1(hashes.SHA256()), 
                                           salt_length=padding.PSS.MAX_LENGTH), 
                               hashes.SHA256())
verifier.update(message)
verifier.verify()

5. Networking


In [ ]:
import requests
r = requests.get('https://www.google.com/imghp')
r.content[:200]

In [ ]:
r.status_code

In [ ]:
r.headers

In [ ]:
len(r.content)

In [ ]:
r.apparent_encoding

In [ ]:
r.elapsed

In [ ]:
r.request.headers

In [ ]:
custom_headers = {"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"}
r = requests.get("https://www.google.com/imghp", headers=custom_headers)
r.request.headers

In [ ]:
import requests
import logging
import http.client

# Enable logging
http.client.HTTPConnection.debuglevel = 1

logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
r = requests.get('https://www.google.com/')

In [ ]:
import urlparse
simple_url = "http://www.example.com/path/to/my/page"
parsed = urlparse.urlparse(simple_url)

In [ ]:
parsed.scheme

In [ ]:
parsed.hostname

In [ ]:
parsed.path

In [ ]:
url_with_query = "http://www.example.com/?page=1&key=Anvn4mo24"
query = urlparse.urlparse(url_with_query).query
urlparse.parse_qs(query)

In [ ]:
import urllib
url = 'https://www.example.com/%5EA-url-with-%-and-%5E?page=page+with%20spaces'
urllib.unquote(url)

In [ ]:
chars = '!@#$%^%$#)'
urllib.quote(chars)

In [ ]:
urllib.unquote_plus(url)

In [ ]:
urllib.quote_plus('one two')

In [ ]:
import requests
from bs4 import BeautifulSoup

http_client.HTTPConnection.debuglevel = 0  # Logging off
r = requests.get("http://www.google.com")
soup = BeautifulSoup(r.content, "lxml")

soup.find_all('p')

In [ ]:
soup.find_all('a')

In [ ]:
for link in soup.find_all('a'):
    print link.text, link["href"]

In [ ]:
import dryscrape
from bs4 import BeautifulSoup
session = dryscrape.Session()
session.visit("http://www.google.com")
r = session.body()
soup = BeautifulSoup(r, "lxml")

In [ ]:
from selenium import webdriver
driver = webdriver.Chrome("/path/to/chromedriver")
driver.get("http://www.google.com")
html = driver.page_source
driver.save_screenshot("screenshot.png")
driver.quit()

In [ ]:
import smtplib

server = smtplib.SMTP('localhost', port=1025)
server.set_debuglevel(True)
server.sendmail("me@localhost", "you@localhost", "This is an email message")
server.quit()

In [ ]:
! host google.com

In [ ]:
! host 172.217.11.14