http://yamalab.tistory.com/80?

  1. Colab에서 Keras를 실행한다. GPU도 써보고 시각화도 해보자
  2. 학습이 완료된 모델을 json, h5와 같은 파일 형태로 저장하자
  3. VM file path에 저장된 모델 파일을 google drive 혹은 local 저장소로 복사하여 저장한다
  4. google API, module을 이용하여 새로운 VM위에 지난 개발에서 학습했던 모델 파일 등을 불러온다
  5. 불러온 파일을 이용하여 하던 개발을 계속하고 다시 1-3을 반복하여 연속성을 만들어준다.

드라이브에 파일 읽기 / 쓰기


In [4]:
!pip install -U -q PyDrive

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file text file.
# 특정 폴더 안으로 파일 삽입
uploaded = drive.CreateFile({'title': 'Sample upload.txt', "parents": [{"kind": "drive#fileLink","id": 'your_drive_id'}]})

uploaded.SetContentString('Sample upload file content')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))

# 3. Load a file by ID and print its contents.
downloaded = drive.CreateFile({'id': uploaded.get('id')})
print('Downloaded content "{}"'.format(downloaded.GetContentString()))


---------------------------------------------------------------------------
ResumableUploadError                      Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/pydrive/files.py in _FilesInsert(self, param)
    368       metadata = self.auth.service.files().insert(**param).execute(
--> 369         http=self.http)
    370     except errors.HttpError as error:

/usr/local/lib/python3.6/dist-packages/googleapiclient/_helpers.py in positional_wrapper(*args, **kwargs)
    129                     logger.warning(message)
--> 130             return wrapped(*args, **kwargs)
    131         return positional_wrapper

/usr/local/lib/python3.6/dist-packages/googleapiclient/http.py in execute(self, http, num_retries)
    810       while body is None:
--> 811         _, body = self.next_chunk(http=http, num_retries=num_retries)
    812       return body

/usr/local/lib/python3.6/dist-packages/googleapiclient/_helpers.py in positional_wrapper(*args, **kwargs)
    129                     logger.warning(message)
--> 130             return wrapped(*args, **kwargs)
    131         return positional_wrapper

/usr/local/lib/python3.6/dist-packages/googleapiclient/http.py in next_chunk(self, http, num_retries)
    915       else:
--> 916         raise ResumableUploadError(resp, content)
    917     elif self._in_error_state:

ResumableUploadError: <HttpError 404 "File not found: your_drive_id">

During handling of the above exception, another exception occurred:

ApiRequestError                           Traceback (most recent call last)
<ipython-input-4-897a4bad690d> in <module>()
     20 
     21 uploaded.SetContentString('Sample upload file content')
---> 22 uploaded.Upload()
     23 print('Uploaded file with ID {}'.format(uploaded.get('id')))
     24 

/usr/local/lib/python3.6/dist-packages/pydrive/files.py in Upload(self, param)
    283         self._FilesPatch(param=param)
    284     else:
--> 285       self._FilesInsert(param=param)
    286 
    287   def Trash(self, param=None):

/usr/local/lib/python3.6/dist-packages/pydrive/auth.py in _decorated(self, *args, **kwargs)
     73       self.http = self.auth.Get_Http_Object()
     74 
---> 75     return decoratee(self, *args, **kwargs)
     76   return _decorated
     77 

/usr/local/lib/python3.6/dist-packages/pydrive/files.py in _FilesInsert(self, param)
    369         http=self.http)
    370     except errors.HttpError as error:
--> 371       raise ApiRequestError(error)
    372     else:
    373       self.uploaded = True

ApiRequestError: <HttpError 404 "File not found: your_drive_id">

3. 케라스 모델파일 읽기/쓰기


In [5]:
from keras.utils import np_utils
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Activation

# mnist import
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()

# 0~1 사이의 값으로 정규화
X_train = X_train.reshape(60000, 784).astype('float32') / 255.0
X_test = X_test.reshape(10000, 784).astype('float32') / 255.0

# 원핫 인코딩
Y_train = np_utils.to_categorical(Y_train)
Y_test = np_utils.to_categorical(Y_test)

model = Sequential()
model.add(Dense(units=64, input_dim=28*28, activation='relu'))
model.add(Dense(units=10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])
hist = model.fit(X_train, Y_train, epochs=2, batch_size=32)

loss_and_metrics = model.evaluate(X_test, Y_test, batch_size=32)

print('loss_and_metrics : ' + str(loss_and_metrics))

# VM local root 경로에 모델파일 저장


model.save('mnist_mlp_model.h5')


Using TensorFlow backend.
Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz
11493376/11490434 [==============================] - 0s 0us/step
Epoch 1/2
60000/60000 [==============================] - 4s 67us/step - loss: 0.6618 - acc: 0.8314
Epoch 2/2
60000/60000 [==============================] - 3s 54us/step - loss: 0.3464 - acc: 0.9026
10000/10000 [==============================] - 0s 27us/step
loss_and_metrics : [0.30194027242064475, 0.9158]

4. root 경로에 저장된 모델파일을 드라이브의 원하는 폴더에 저장


In [6]:
# 1. Authenticate and create the PyDrive client.

auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file text file.
# 특정 폴더 안으로 파일 삽입
uploaded = drive.CreateFile({'title': 'mnist_mlp_model.h5', "parents": [{"kind": "drive#fileLink","id": 'your_drive_id'}]})
uploaded.SetContentString('Sample upload file content')
uploaded.SetContentFile('mnist_mlp_model.h5')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))

# 3. Load a file by ID and print its contents.
downloaded = drive.CreateFile({'id': uploaded.get('id')})


 코드를 실행하면, 앞선 과정에서 저장했던 h5 파일이 드라이브의  폴더 안에 저장된다.


  File "<ipython-input-6-8aca782b1313>", line 22
    위 코드를 실행하면, 앞선 과정에서 저장했던 h5 파일이 드라이브의 내 폴더 안에 저장된다.
        ^
SyntaxError: invalid syntax

In [9]:
#- REST API로 드라이브에 있는 모델파일을 BytesIO로 다운로드

from google.colab import auth
auth.authenticate_user()

from googleapiclient.discovery import build
drive_service = build('drive', 'v3')

import io
from io import BytesIO   
from googleapiclient.http import MediaIoBaseDownload

request = drive_service.files().get_media(fileId='11RMJNeeZLgUtuuvmyq3LkLBTnTc4vhHh')
downloaded = io.BytesIO()
downloader = MediaIoBaseDownload(downloaded, request)
done = False
while done is False:
  status, done = downloader.next_chunk()
  if status:
      print("Download %%%d%%." % int(status.progress() * 100))
  print("Download Complete!")

downloaded.seek(0)

with open('/tmp/mnist_mlp_model.h5', 'wb') as f:
    f.write(downloaded.read())



#REST API를 이용해 위에서 저장했던 파일을 다운받는다. 이 파일의 형태는 buffer의 형태이므로, VM의 tmp 폴더에 저장해준다.


---------------------------------------------------------------------------
HttpError                                 Traceback (most recent call last)
<ipython-input-9-85e906912dd0> in <module>()
     15 done = False
     16 while done is False:
---> 17   status, done = downloader.next_chunk()
     18   if status:
     19       print("Download %%%d%%." % int(status.progress() * 100))

/usr/local/lib/python3.6/dist-packages/googleapiclient/_helpers.py in positional_wrapper(*args, **kwargs)
    128                 elif positional_parameters_enforcement == POSITIONAL_WARNING:
    129                     logger.warning(message)
--> 130             return wrapped(*args, **kwargs)
    131         return positional_wrapper
    132 

/usr/local/lib/python3.6/dist-packages/googleapiclient/http.py in next_chunk(self, num_retries)
    692       return MediaDownloadProgress(self._progress, self._total_size), self._done
    693     else:
--> 694       raise HttpError(resp, content, uri=self._uri)
    695 
    696 

HttpError: <HttpError 404 when requesting https://www.googleapis.com/drive/v3/files/11RMJNeeZLgUtuuvmyq3LkLBTnTc4vhHh?alt=media returned "File not found: 11RMJNeeZLgUtuuvmyq3LkLBTnTc4vhHh.">

In [10]:
from keras.models import load_model

loaded_model = load_model('/tmp/mnist_mlp_model.h5')


#그리고 tmp 폴더에 저장해둔 파일을 load하면 한 사이클이 끝이 난다.


---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-10-afa82365a8bb> in <module>()
      1 from keras.models import load_model
      2 
----> 3 loaded_model = load_model('/tmp/mnist_mlp_model.h5')
      4 
      5 

/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_model(filepath, custom_objects, compile)
    415     model = None
    416     opened_new_file = not isinstance(filepath, h5py.Group)
--> 417     f = h5dict(filepath, 'r')
    418     try:
    419         model = _deserialize_model(f, custom_objects, compile)

/usr/local/lib/python3.6/dist-packages/keras/utils/io_utils.py in __init__(self, path, mode)
    184             self._is_file = False
    185         elif isinstance(path, str):
--> 186             self.data = h5py.File(path, mode=mode)
    187             self._is_file = True
    188         elif isinstance(path, dict):

/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py in __init__(self, name, mode, driver, libver, userblock_size, swmr, **kwds)
    310             with phil:
    311                 fapl = make_fapl(driver, libver, **kwds)
--> 312                 fid = make_fid(name, mode, userblock_size, fapl, swmr=swmr)
    313 
    314                 if swmr_support:

/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py in make_fid(name, mode, userblock_size, fapl, fcpl, swmr)
    140         if swmr and swmr_support:
    141             flags |= h5f.ACC_SWMR_READ
--> 142         fid = h5f.open(name, flags, fapl=fapl)
    143     elif mode == 'r+':
    144         fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/h5f.pyx in h5py.h5f.open()

OSError: Unable to open file (unable to open file: name = '/tmp/mnist_mlp_model.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)

In [12]:
## predict


from numpy import argmax

xhat_idx = np.random.choice(X_test.shape[0], 5)
xhat = X_test[xhat_idx]
yhat = loaded_model.predict_classes(xhat)

for i in range(5):
    print('True : ' + str(argmax(Y_test[xhat_idx[i]])) + ', Predict : ' + str(yhat[i]))


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-12-d8e060bb615f> in <module>()
      6 xhat_idx = np.random.choice(X_test.shape[0], 5)
      7 xhat = X_test[xhat_idx]
----> 8 yhat = loaded_model.predict_classes(xhat)
      9 
     10 for i in range(5):

NameError: name 'loaded_model' is not defined

In [0]:


In [0]: