Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit405c691

Browse files
add regularization and gradient clipping
1 parent4c28c6f commit405c691

File tree

1 file changed

+75
-60
lines changed

1 file changed

+75
-60
lines changed

‎python/dnlp/core/dnn_crf.py‎

Lines changed: 75 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99

1010
classDnnCrf(DnnCrfBase):
1111
def__init__(self,*,config:DnnCrfConfig=None,task='cws',data_path:str='',dtype:type=tf.float32,
12-
mode:str='train',predict:str='ll',nn:str,model_path:str='',embedding_path:str=''):
12+
mode:str='train',dropout_position:str='input',predict:str='ll',nn:str,model_path:str='',
13+
embedding_path:str='',remark:str=''):
1314
ifmodenotin ['train','predict']:
1415
raiseException('mode error')
1516
ifnnnotin ['mlp','rnn','lstm','bilstm','gru']:
@@ -20,63 +21,75 @@ def __init__(self, *, config: DnnCrfConfig = None, task='cws', data_path: str =
2021
self.mode=mode
2122
self.task=task
2223
self.nn=nn
24+
self.remark=remark
2325
self.embedding_path=embedding_path
24-
25-
# 构建
26-
tf.reset_default_graph()
27-
self.transition=self.__get_variable([self.tags_count,self.tags_count],'transition')
28-
self.transition_init=self.__get_variable([self.tags_count],'transition_init')
29-
self.params= [self.transition,self.transition_init]
30-
# 输入层
31-
ifmode=='train':
32-
self.input=tf.placeholder(tf.int32, [self.batch_size,self.batch_length,self.windows_size])
33-
self.real_indices=tf.placeholder(tf.int32, [self.batch_size,self.batch_length])
34-
else:
35-
self.input=tf.placeholder(tf.int32, [None,self.windows_size])
36-
37-
self.seq_length=tf.placeholder(tf.int32, [None])
38-
39-
# 查找表层
40-
self.embedding_layer=self.get_embedding_layer()
41-
# 执行drpout
42-
self.embedding_layer=self.get_dropout_layer(self.embedding_layer)
43-
# 隐藏层
44-
ifnn=='mlp':
45-
self.hidden_layer=self.get_mlp_layer(self.embedding_layer)
46-
elifnn=='lstm':
47-
self.hidden_layer=self.get_lstm_layer(self.embedding_layer)
48-
elifnn=='bilstm':
49-
self.hidden_layer=self.get_bilstm_layer(self.embedding_layer)
50-
elifnn=='gru':
51-
self.hidden_layer=self.get_gru_layer(self.embedding_layer)
52-
else:
53-
self.hidden_layer=self.get_rnn_layer(self.embedding_layer)
54-
# 输出层
55-
self.output=self.get_output_layer(self.hidden_layer)
56-
57-
ifmode=='predict':
58-
ifpredict!='ll':
59-
self.output=tf.squeeze(tf.transpose(self.output),axis=2)
60-
self.seq,self.best_score=tf.contrib.crf.crf_decode(self.output,self.transition,self.seq_length)
61-
self.sess=tf.Session()
62-
self.sess.run(tf.global_variables_initializer())
63-
tf.train.Saver().restore(save_path=self.model_path,sess=self.sess)
64-
else:
65-
self.loss,_=tf.contrib.crf.crf_log_likelihood(self.output,self.real_indices,self.seq_length,
66-
self.transition)
67-
self.optimizer=tf.train.AdagradOptimizer(self.learning_rate)
68-
self.new_optimizer=tf.train.AdamOptimizer()
69-
self.train=self.optimizer.minimize(-self.loss)
70-
current_dir=os.path.dirname(__file__)
71-
dest_dir=os.path.realpath(os.path.join(current_dir,'..\\data\\logs'))
72-
self.train_writer=tf.summary.FileWriter(dest_dir ,flush_secs=10)
73-
tf.summary.scalar('loss',-self.loss)
74-
# tf.summary.scalar('accuracy',)
75-
self.merged=tf.summary.merge_all()
76-
26+
self.graph=tf.Graph()
27+
withself.graph.as_default():
28+
# 构建
29+
# tf.reset_default_graph()
30+
self.transition=self.__get_variable([self.tags_count,self.tags_count],'transition')
31+
self.transition_init=self.__get_variable([self.tags_count],'transition_init')
32+
self.params= [self.transition,self.transition_init]
33+
# 输入层
34+
35+
ifmode=='train':
36+
self.input=tf.placeholder(tf.int32, [self.batch_size,self.batch_length,self.windows_size])
37+
self.real_indices=tf.placeholder(tf.int32, [self.batch_size,self.batch_length])
38+
else:
39+
self.input=tf.placeholder(tf.int32, [None,self.windows_size])
40+
41+
self.seq_length=tf.placeholder(tf.int32, [None])
42+
43+
# 查找表层
44+
self.embedding_layer=self.get_embedding_layer()
45+
# 执行drpout
46+
ifdropout_position=='input':
47+
self.embedding_layer=self.get_dropout_layer(self.embedding_layer)
48+
# 隐藏层
49+
ifnn=='mlp':
50+
self.hidden_layer=self.get_mlp_layer(self.embedding_layer)
51+
elifnn=='lstm':
52+
self.hidden_layer=self.get_lstm_layer(self.embedding_layer)
53+
elifnn=='bilstm':
54+
self.hidden_layer=self.get_bilstm_layer(self.embedding_layer)
55+
elifnn=='gru':
56+
self.hidden_layer=self.get_gru_layer(self.embedding_layer)
57+
else:
58+
self.hidden_layer=self.get_rnn_layer(self.embedding_layer)
59+
ifdropout_position=='hidden':
60+
self.hidden_layer=self.get_dropout_layer(self.hidden_layer)
61+
# 输出层
62+
self.output=self.get_output_layer(self.hidden_layer)
63+
64+
ifmode=='predict':
65+
ifpredict!='ll':
66+
self.output=tf.squeeze(tf.transpose(self.output),axis=2)
67+
self.seq,self.best_score=tf.contrib.crf.crf_decode(self.output,self.transition,self.seq_length)
68+
self.sess=tf.Session()
69+
self.sess.run(tf.global_variables_initializer())
70+
tf.train.Saver().restore(save_path=self.model_path,sess=self.sess)
71+
else:
72+
self.crf_loss,_=tf.contrib.crf.crf_log_likelihood(self.output,self.real_indices,self.seq_length,
73+
self.transition)
74+
#self.loss = -self.loss
75+
self.regularization=tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(self.lam),
76+
self.params )
77+
self.loss=-self.crf_loss/self.batch_size+self.regularization
78+
self.optimizer=tf.train.AdagradOptimizer(self.learning_rate)
79+
self.new_optimizer=tf.train.AdamOptimizer()
80+
gvs=self.optimizer.compute_gradients(self.loss)
81+
cliped_grad= [(tf.clip_by_norm(grad,5)ifgradisnotNoneelsegrad,var)forgrad,varingvs]
82+
self.train=self.optimizer.apply_gradients(cliped_grad)# self.optimizer.minimize(self.loss)
83+
# self.train = self.optimizer.minimize(self.loss)
84+
current_dir=os.path.dirname(__file__)
85+
dest_dir=os.path.realpath(os.path.join(current_dir,'..\\data\\logs'))
86+
self.train_writer=tf.summary.FileWriter(dest_dir,flush_secs=10)
87+
self.mean_loss=tf.reduce_mean(self.loss)
88+
tf.summary.scalar('loss',self.mean_loss)
89+
self.merged=tf.summary.merge_all()
7790

7891
deffit(self,epochs:int=50,interval:int=10):
79-
withtf.Session()assess:
92+
withtf.Session(graph=self.graph)assess:
8093
tf.global_variables_initializer().run()
8194
saver=tf.train.Saver(max_to_keep=epochs)
8295
forepochinrange(1,epochs+1):
@@ -85,13 +98,15 @@ def fit(self, epochs: int = 50, interval: int = 10):
8598
foriinrange(self.batch_count):
8699
characters,labels,lengths=self.get_batch()
87100
feed_dict= {self.input:characters,self.real_indices:labels,self.seq_length:lengths}
88-
_,summary,loss=sess.run([self.train,self.merged,-self.loss],feed_dict=feed_dict)
89-
# summary,loss = sess.run([],feed_dict=feed_dict)
90-
self.train_writer.add_summary(summary,j)
91-
j+=1
101+
_,summary,loss=sess.run([self.train,self.merged,self.mean_loss],feed_dict=feed_dict)
102+
self.train_writer.add_summary(summary,j)
103+
j+=1
92104
ifepoch%interval==0:
93105
ifnotself.embedding_path:
94-
model_path='../dnlp/models/{0}-{1}-{2}.ckpt'.format(self.task,self.nn,epoch)
106+
ifself.remark:
107+
model_path='../dnlp/models/{0}-{1}-{2}-{3}.ckpt'.format(self.task,self.nn,self.remark,epoch)
108+
else:
109+
model_path='../dnlp/models/{0}-{1}-{2}.ckpt'.format(self.task,self.nn,epoch)
95110
else:
96111
model_path='../dnlp/models/{0}-{1}-embedding-{2}.ckpt'.format(self.task,self.nn,epoch)
97112
saver.save(sess,model_path)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp