[실습] softmax model, train, validation, learning rate 조절, data processing

https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-07_1_tips.ipynb

 

Training data and Dataset

x_train = torch.FloatTensor([[1, 2, 1],
                             [1, 3, 2],
                             [1, 3, 4],
                             [1, 5, 5],
                             [1, 7, 5],
                             [1, 2, 5],
                             [1, 6, 6],
                             [1, 7, 7]
                            ])
y_train = torch.LongTensor([2, 2, 2, 1, 1, 1, 0, 0])

x_test = torch.FloatTensor([[2, 1, 1], [3, 1, 2], [3, 3, 4]])
y_test = torch.LongTensor([2, 2, 2])

|x_train| = (m, 3)

|y_train| = (m, )

|x_test| = (m', 3)

|y_test| = (m', )

같은 분포로부터 얻어진 데이터임

 

 

Model

class SoftmaxClassifierModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(3, 3)
    def forward(self, x):
        return self.linear(x)
        
model = SoftmaxClassifierModel() # 모델 선언
optimizer = optim.SGD(model.parameters(), lr=0.1) # optimizer 설정

3개의 element를 가진 1d vector을 받아서 3개의 element를 가진 1d vector로 return해줌

|x| = (m,3) #x의 사이즈 를 받아서 (m,3)으로 return

 

 

Train

def train(model, optimizer, x_train, y_train):
    nb_epochs = 20
    for epoch in range(nb_epochs):

        # H(x) 계산
        prediction = model(x_train)
        # |x_train| = (m,3) |prediction| = (m.3)

        # cost 계산
        cost = F.cross_entropy(prediction, y_train)
        # |y_train| = (m, )

        # cost로 H(x) 개선
        optimizer.zero_grad()
        cost.backward()
        optimizer.step()

        print('Epoch {:4d}/{} Cost: {:.6f}'.format(
            epoch, nb_epochs, cost.item()
        ))

train(model, optimizer, x_train, y_train)

 

 

Test (Validation)

def test(model, optimizer, x_test, y_test):
    prediction = model(x_test)
    predicted_classes = prediction.max(1)[1]
    # |x_test| = (m', 3)  |prediction| = (m', 3)
    correct_count = (predicted_classes == y_test).sum().item()
    cost = F.cross_entropy(prediction, y_test)

    print('Accuracy: {}% Cost: {:.6f}'.format(
         correct_count / len(y_test) * 100, cost.item()
    ))

test(model, optimizer, x_test, y_test)

 

 

Learning rate 

model = SoftmaxClassifierModel()
optimizer = optim.SGD(model.parameters(), lr=1e5) #여기서 lr값 바꾸기
train(model, optimizer, x_train, y_train)

 

 

Data Perprocessing (데이터 전처리)

여기서 사용한 방식은 standardization 방식임 (정규분포로 만들어주는 것)

σ  는 standard deviation,  μ  는 평균값

x_train = torch.FloatTensor([[73, 80, 75],
                             [93, 88, 93],
                             [89, 91, 90],
                             [96, 98, 100],
                             [73, 66, 70]])
y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])
# |x_train| = (m,3) |y_train| = (m,)

mu = x_train.mean(dim=0)
sigma = x_train.std(dim=0)
norm_x_train = (x_train - mu) / sigma

 

 

 

Regularization

overfitting을 방지하기 위해 w(가중치)의 값이 너무 크지 않도록 규제

def train_with_regularization(model, optimizer, x_train, y_train):
    nb_epochs = 20
    for epoch in range(nb_epochs):

        # H(x) 계산
        prediction = model(x_train)

        # cost 계산
        cost = F.mse_loss(prediction, y_train)
        
        # l2 norm 계산
        l2_reg = 0
        for param in model.parameters():
            l2_reg += torch.norm(param)
            
        cost += l2_reg

        # cost로 H(x) 개선
        optimizer.zero_grad()
        cost.backward()
        optimizer.step()

        print('Epoch {:4d}/{} Cost: {:.6f}'.format(
            epoch+1, nb_epochs, cost.item()
        ))