In [230]:
import numpy as np
import pandas as pd
import os
import glob
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import math
In [231]:
DATA_DIR = "img_new"
TEST_DIR = "Data_Uji"
TRAIN_TEST_SPLIT = 0.7
IM_WIDTH = IM_HEIGHT = 198
ID_TIPE_MAP = {0: 'Phospor', 1: 'Kalium', 2: 'Nitrogen', 3: 'Normal'}
TIPE_ID_MAP = dict((r, i) for i, r in ID_TIPE_MAP.items())
UJI_ID_TIPE_MAP = {0: 'Phospor', 1: 'Kalium', 2: 'Nitrogen', 3: 'Normal'}
UJI_TIPE_ID_MAP = dict((r, i) for i, r in UJI_ID_TIPE_MAP.items())
ID_TIPE_MAP, TIPE_ID_MAP
Out[231]:
In [232]:
def uk(img):
#print(img)
uk1=len(img)
uk2=len(img[0])
uk3=''
try:
m=len(img[0][0])
uk3="x"+str(m)
except:
uk3=0
uk3=""
print(type(img))
print(str(uk1)+"x"+str(uk2)+uk3)
return "OK"
def parse_filepath(filepath):
try:
path, filename = os.path.split(filepath)
filename, ext = os.path.splitext(filename)
tipe, _ = filename.split("_")
return ID_TIPE_MAP[int(tipe)]
except Exception as e:
print('None:',filepath)
return None
def uji_parse_filepath(filepath):
try:
path, filename = os.path.split(filepath)
filename, ext = os.path.splitext(filename)
tipe, _ = filename.split("_")
return UJI_ID_TIPE_MAP[int(tipe)]
except Exception as e:
print('None:',filepath)
return None
def uji_parse_filepath0(filepath):
try:
path, filename = os.path.split(filepath)
filename, ext = os.path.splitext(filename)
uji_tipe=0
return UJI_ID_TIPE_MAP[uji_tipe]
except Exception as e:
print('None:',filepath)
return None
In [233]:
files = glob.glob(os.path.join(DATA_DIR, "*.jpg"))
attributes = list(map(parse_filepath, files))
df = pd.DataFrame(attributes)
df['file'] = files
df.columns = ['tipe', 'file']
df = df.dropna()
df.head()
df.describe()
df['tipe'].describe()
df.groupby(by=['tipe']).count().plot(kind='bar')
p = np.random.permutation(len(df))
df['tipe_id'] = df['tipe'].map(lambda ztipe: TIPE_ID_MAP[ztipe])
print('Banyak Data Training:',len(p))
all_idx=p
df.head()
#df.tail()
Out[233]:
Membentuk Matrik
In [235]:
from keras.utils import to_categorical
from PIL import Image
def get_data_generator(df, indices, for_training, batch_size=16):
images, ltipe = [], []
while True:
for i in indices:
r = df.iloc[i]
file, t= r['file'],r['tipe_id']
im = Image.open(file)
im = im.resize((IM_WIDTH, IM_HEIGHT))
im = np.array(im) / 255.0
images.append(im)
ltipe.append(to_categorical(t, len(TIPE_ID_MAP)))
if len(images) >= batch_size:
yield np.array(images), [np.array(ltipe)]
images,ltipe = [],[]
if not for_training:
print("No")
break
Model Convolution¶
In [236]:
from keras.layers import Input, Dense, BatchNormalization, Conv2D, MaxPool2D, GlobalMaxPool2D, Dropout
from keras.optimizers import SGD
from keras.models import Model
def conv_block(inp, filters=32, bn=True, pool=True):
_ = Conv2D(filters=filters, kernel_size=3, activation='relu')(inp)
if bn:
_ = BatchNormalization()(_)
if pool:
_ = MaxPool2D()(_)
return _
input_layer = Input(shape=(IM_HEIGHT, IM_WIDTH, 3))
_ = conv_block(input_layer, filters=32, bn=False, pool=False)
_ = conv_block(_, filters=32*2)
_ = conv_block(_, filters=32*3)
_ = conv_block(_, filters=32*4)
_ = conv_block(_, filters=32*5)
_ = conv_block(_, filters=32*6)
bottleneck = GlobalMaxPool2D()(_)
# for tipe prediction
_ = Dense(units=128, activation='relu')(bottleneck)
tipe_output = Dense(units=len(TIPE_ID_MAP), activation='softmax', name='tipe_output')(_)
model = Model(inputs=input_layer, outputs=[tipe_output])
model.compile(optimizer='rmsprop',
loss={'tipe_output': 'categorical_crossentropy'},
loss_weights={'tipe_output': 1.5},
metrics={'tipe_output': 'accuracy'})
model.summary()
In [ ]:
Pemodelan Data Training¶
In [237]:
from keras.callbacks import ModelCheckpoint
data_size =len(all_idx)
all_gen = get_data_generator(df, all_idx, for_training=True, batch_size=data_size)
valid_gen = get_data_generator(df, all_idx, for_training=True, batch_size=data_size)
callbacks = [
ModelCheckpoint("./model_checkpoint", monitor='val_loss')
]
print(all_gen)
print(valid_gen)
GENERATE = model.fit_generator(all_gen,steps_per_epoch=len(all_idx)/data_size,epochs=5,callbacks=callbacks,validation_data=valid_gen,validation_steps= len(all_idx)/data_size)
#GENERATE = model.fit_generator(all_gen,steps_per_epoch=1,epochs=1,callbacks=callbacks,validation_data=valid_gen,validation_steps=1)
print(model.metrics_names)
print(GENERATE)
In [238]:
MM=model.evaluate_generator(all_gen, steps=11) # steps=len(test_idx)//128
dict(zip(model.metrics_names, MM))
Out[238]:
In [240]:
x_test, tipe_true= next(all_gen)
tipe_pred= model.predict_on_batch(x_test)
print(len(tipe_true) ,'x', len(tipe_true[0])) #1 x 25 <class 'list'>
print(len(tipe_pred),'x',len(tipe_pred[0])) #25 x 4 <class 'numpy.ndarray'>
print(len(x_test),'x',len(x_test[0])) #25 x 198 <class 'numpy.ndarray'>
In [241]:
LT1=tipe_true[0].tolist()
AT1=np.asarray(tipe_true[0], dtype=np.float32)
LT2=tipe_pred.tolist()
AT2=tipe_pred
In [242]:
tipe_true =AT1.argmax(axis=-1)
tipe_pred =AT2.argmax(axis=-1)
In [243]:
from sklearn.metrics import classification_report
print("Olah Data:")
print(classification_report(tipe_true,tipe_pred))
In [244]:
import math
training_n =data_size # len(all_idx) #Sebanyak Data Training
random_indices = np.random.permutation(training_n)
benar=0
no=0
for i, img_idx in enumerate(random_indices):
no=no+1
T=tipe_true[img_idx]
P=tipe_pred[img_idx]
V='No'
if T==P:
V='Yes'
benar=benar+1
print(str(no), ' ',str(img_idx),' ', str(T),' ', str(P),' ', str(V))
print('Benar:',str(benar), ' dari ',str(no),' dataUji')
acc=benar/n
print('Acc:',str(acc))
In [261]:
#SHOW DATA TRAINING
plt.clf()
n_cols = 5
n_rows = math.ceil(training_n / n_cols)
no=0
fig, axes = plt.subplots(n_rows, n_cols, figsize=(15, 20))
for i, img_idx in enumerate(random_indices):
no=no+1
T=tipe_true[img_idx]
P=tipe_pred[img_idx]
ax = axes.flat[i]
ax.imshow(x_test[img_idx])
#ax.set_title('{}'.format(ID_TIPE_MAP[P]))
ax.set_xlabel('{}.{} ? {}'.format(str(no), ID_TIPE_MAP[T],ID_TIPE_MAP[P]))
ax.set_xticks([])
ax.set_yticks([])
Pemodelan UJI¶
In [249]:
uji_files = glob.glob(os.path.join(TEST_DIR, "*.jpg"))
uji_attributes = list(map(uji_parse_filepath, uji_files))
uji_df = pd.DataFrame(uji_attributes)
uji_df['file'] = uji_files
uji_df.columns = ['tipe', 'file']
uji_df = uji_df.dropna()
uji_df.head()
uji_df.describe()
uji_df['tipe'].describe()
uji_df.groupby(by=['tipe']).count().plot(kind='bar')
uji_p = np.random.permutation(len(uji_df))
uji_df['tipe_id'] = uji_df['tipe'].map(lambda xtipe: UJI_TIPE_ID_MAP[xtipe])
print('Banyak Data Uji:',len(uji_p))
uji_idx=uji_p
uji_df.head()
#uji_df.tail()
Out[249]:
In [250]:
#UJI DATA UJI
uji_data_size=len(uji_idx)
uji_gen = get_data_generator(uji_df, uji_idx, for_training=True, batch_size=uji_data_size)
#GENERATE_UJI = model.fit_generator(uji_gen,steps_per_epoch=len(uji_idx)/uji_data_size,epochs=5,callbacks=callbacks,validation_data=valid_gen,validation_steps= len(uji_idx)/uji_data_size)
GENERATE_UJI = model.fit_generator(uji_gen,steps_per_epoch=1,epochs=1,callbacks=callbacks,validation_data=valid_gen,validation_steps=1)
In [252]:
print(model.metrics_names)
print(uji_gen)
UJI_M=model.evaluate_generator(uji_gen, steps=11) # steps=len(test_idx)//128
dict(zip(model.metrics_names, UJI_M))
Out[252]:
In [253]:
uji_x_test, uji_tipe_true= next(uji_gen)
uji_tipe_pred= model.predict_on_batch(uji_x_test)
print(len(uji_tipe_true) ,'x', len(uji_tipe_true[0])) #1 x 25 <class 'list'>
print(len(uji_tipe_pred),'x',len(uji_tipe_pred[0])) #25 x 4 <class 'numpy.ndarray'>
print(len(uji_x_test),'x',len(uji_x_test[0])) #25 x 198 <class 'numpy.ndarray'>
LT1=uji_tipe_true[0].tolist()
AT1=np.asarray(uji_tipe_true[0], dtype=np.float32)
LT2=uji_tipe_pred.tolist()
AT2=uji_tipe_pred
uji_tipe_true =AT1.argmax(axis=-1)
uji_tipe_pred =AT2.argmax(axis=-1)
from sklearn.metrics import classification_report
print("Olah Data Uji:")
print(classification_report(uji_tipe_true,uji_tipe_pred))
In [254]:
uji_n =uji_data_size # len(all_idx) #Sebanyak Data Training
uji_random_indices = np.random.permutation(uji_n)
benar=0
no=0
for i, uji_img_idx in enumerate(uji_random_indices):
no=no+1
T=uji_tipe_true[uji_img_idx]
P=uji_tipe_pred[uji_img_idx]
V='No'
if T==P:
V='Yes'
benar=benar+1
print(str(no), ' ',str(uji_img_idx),' ', str(T),' ', str(P),' ', str(V))
print('Benar:',str(benar), ' dari ',str(no),' dataUji')
acc=benar/n
print('Acc Uji:',str(acc))
In [260]:
#SHOW DATA TRAINING
plt.clf()
n_cols = 5
n_rows = math.ceil(uji_n / n_cols)
no=0
fig, axes = plt.subplots(n_rows, n_cols, figsize=(15, 20))
for i, uji_img_idx in enumerate(uji_random_indices):
no=no+1
T=uji_tipe_true[uji_img_idx]
P=uji_tipe_pred[uji_img_idx]
ax = axes.flat[i]
ax.imshow(uji_x_test[uji_img_idx])
#ax.set_title('{}'.format(UJI_ID_TIPE_MAP[P]))
ax.set_xlabel('{}.{} ? {}'.format(str(no), UJI_ID_TIPE_MAP[T],UJI_ID_TIPE_MAP[P]))
ax.set_xticks([])
ax.set_yticks([])
In [ ]:
