Skip to content

Commit 2428a3b

Browse files
hailiang-wanghailiang-wang
committed
chatopera/docs#78 enable train and wait for status
1 parent 04ab5db commit 2428a3b

5 files changed

Lines changed: 474 additions & 4 deletions

File tree

‎src/main/java/com/chatopera/bot/sdk/TrainMgr.java‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/*
2+
* Copyright (C) 2018-2023 Chatopera Inc, <https://www.chatopera.com>
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.chatopera.bot.sdk;
18+
19+
import com.chatopera.bot.exception.ChatbotException;
20+
import com.chatopera.bot.exception.ResourceInvalidException;
21+
import com.chatopera.bot.exception.ResourceOperationException;
22+
import com.chatopera.bot.sdk.basics.Response;
23+
import com.chatopera.bot.sdk.models.Status;
24+
import com.chatopera.bot.utils.Logger;
25+
import org.apache.commons.lang3.StringUtils;
26+
import org.json.JSONObject;
27+
28+
import java.net.MalformedURLException;
29+
30+
/**
31+
* 训练管理
32+
*/
33+
public class TrainMgr {
34+
35+
private Chatbot chatbot;
36+
37+
// 不支持定义空实例
38+
private TrainMgr() {
39+
}
40+
41+
public TrainMgr(final String clientId, final String clientSecret, final String baseUrl) throws MalformedURLException, ChatbotException, MalformedURLException, ChatbotException {
42+
this.chatbot = new Chatbot(clientId, clientSecret, baseUrl);
43+
}
44+
45+
public TrainMgr(final String clientId, final String clientSecret) throws MalformedURLException, ChatbotException {
46+
this.chatbot = new Chatbot(clientId, clientSecret);
47+
}
48+
49+
/**
50+
* Get Chatbot Status
51+
*
52+
* @return
53+
*/
54+
public Status getStatus() throws ChatbotException, ResourceInvalidException {
55+
Response resp = this.chatbot.command("GET", "/clause/devver/build");
56+
if (resp.getRc() == 0) {
57+
Status status = new Status(resp.getStatus());
58+
return status;
59+
} else {
60+
throw new ChatbotException("Unexpected status result. " + (StringUtils.isNotBlank(resp.getError()) ? resp.getError() : ""));
61+
}
62+
}
63+
64+
/**
65+
* 训练多轮对话脚本
66+
*
67+
* @return
68+
*/
69+
protected boolean trainConversations() throws ChatbotException, ResourceInvalidException {
70+
Status currentStatus = getStatus();
71+
72+
if (currentStatus.getReparse() != 0) {
73+
// 多轮对话待同步
74+
Response resp = this.chatbot.command("POST", "/conversation/sync/customdicts");
75+
return resp.getRc() == 0;
76+
} else {
77+
// 服务器端机器人多轮对话已经同步最新词典
78+
return true;
79+
}
80+
}
81+
82+
/**
83+
* 训练意图识别，返回提交结果，该任务提交成功后，异步执行。
84+
*
85+
* @return true: 开始执行训练（此时服务器在执行任务，训练时间取决于数据量）；false：不能正常执行训练
86+
* @throws ChatbotException
87+
* @throws ResourceInvalidException
88+
*/
89+
protected boolean trainIntents() throws ChatbotException, ResourceInvalidException, ResourceOperationException {
90+
Status currentStatus = getStatus();
91+
92+
if (currentStatus.getRetrain() != 0) {
93+
// 意图识别模型待训练
94+
Response resp = this.chatbot.command("POST", "/clause/devver/train");
95+
if (resp.getRc() == 0) {
96+
// 提交并开始执行训练
97+
return true;
98+
} else if (resp.getRc() == 21 || resp.getRc() == 22 || resp.getRc() == 24) {
99+
Logger.warn("[trainIntents] 没有意图或意图没有说法，此时不需要训练。");
100+
return true;
101+
} else if (resp.getRc() == 25) {
102+
Logger.warn("[trainIntents] 存在不合法的词典信息，无法开始训练");
103+
return false;
104+
} else {
105+
throw new ResourceOperationException("[trainIntents] Unexpected operation results.");
106+
}
107+
} else {
108+
// 训练任务未开始，因为服务器端意图识别模型已经同步最新的训练数据
109+
// 不需要重新训练
110+
return true;
111+
}
112+
}
113+
114+
115+
/**
116+
* 训练知识库，该任务提交成功后，异步执行。
117+
*
118+
* @return 训练任务是否启动。
119+
*/
120+
protected boolean trainFAQs() throws ResourceInvalidException, ChatbotException {
121+
Status currentStatus = getStatus();
122+
123+
if (currentStatus.getReindex() != 0) {
124+
// 多轮对话待同步
125+
Response resp = this.chatbot.command("POST", "/faq/sync/customdicts");
126+
return resp.getRc() == 0;
127+
} else {
128+
// 服务器端机器人知识库索引已经与最新词典信息一致，不需要重新训练
129+
return true;
130+
}
131+
}
132+
133+
/**
134+
* Start in parallel, train all domains
135+
*/
136+
public void trainAll() {
137+
138+
try {
139+
this.trainConversations();
140+
} catch (Exception e) {
141+
Logger.trace("[trainAll] unexpected result to start train conversation.");
142+
e.printStackTrace();
143+
}
144+
145+
try {
146+
this.trainFAQs();
147+
} catch (Exception e) {
148+
Logger.trace("[trainAll] unexpected result to start train faqs.");
149+
e.printStackTrace();
150+
}
151+
152+
try {
153+
this.trainIntents();
154+
} catch (Exception e) {
155+
Logger.trace("[trainAll] unexpected result to start train intents.");
156+
e.printStackTrace();
157+
}
158+
}
159+
160+
/**
161+
* 等待训练任务结束
162+
*/
163+
public void waitForJobsDone() throws InterruptedException, ResourceInvalidException, ChatbotException {
164+
while (!isUpdated()) {
165+
Logger.warn("TrainMgr [waitForJobsDone] still in progress ...");
166+
Thread.sleep(10000);
167+
}
168+
}
169+
170+
171+
/**
172+
* 检查机器人的训练任务是否已经结束
173+
*
174+
* @return
175+
*/
176+
private boolean isUpdated() throws ResourceInvalidException, ChatbotException {
177+
final Status status = getStatus();
178+
179+
boolean isUpdatedIntents = true;
180+
boolean isUpdatedFAQs = true;
181+
182+
/**
183+
* 检查多轮对话状态
184+
* 多轮对话的重新训练不是异步的，在提交了训练任务后，是立即返回结果的
185+
*/
186+
boolean isUpdatedConversations = true;
187+
188+
/**
189+
* 检查意图识别状态
190+
*/
191+
if (status.getRetrain() == 1) {
192+
// 同步中
193+
isUpdatedIntents = false;
194+
} else {
195+
// 以下情况，返回 true
196+
// * 意图识别完成同步
197+
// * 训练过程中，意图识别数据发生变化，本次训练不有意义
198+
isUpdatedIntents = true;
199+
}
200+
201+
/**
202+
* 检查 FAQs 状态
203+
*/
204+
if (status.getReindex() == 1) {
205+
// 正在更新
206+
isUpdatedFAQs = false;
207+
} else {
208+
// 以下情况，返回 true
209+
// * 已经同步
210+
// * 知识库或自定义词典变更，无法继续同步，可返回后重新提交。
211+
isUpdatedFAQs = true;
212+
}
213+
214+
215+
return isUpdatedIntents && isUpdatedFAQs && isUpdatedConversations;
216+
}
217+
218+
219+
}

‎src/main/java/com/chatopera/bot/sdk/models/Status.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Copyright (C) 2018-2023 Chatopera Inc, <https://www.chatopera.com>
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.chatopera.bot.sdk.models;
18+
19+
import com.chatopera.bot.exception.ResourceInvalidException;
20+
import org.json.JSONObject;
21+
22+
/**
23+
* 机器人状态
24+
*/
25+
public class Status {
26+
27+
/**
28+
* https://gitlab.chatopera.com/chatopera/chatopera.bot/issues/554
29+
* 状态的定义
30+
* | rc | 含义 |
31+
* | --- | --- |
32+
* | 0 | 模型完成构建 |
33+
* | 1 | 参数错误，比如没有传chabotID，在dashboard中调用不可能返回这个错误 |
34+
* | 2 | 正在训练中 |
35+
* | 3 | 训练失败，此处没有给出原因 |
36+
* | 4 | 训练失败，没有训练内容，比如没有意图数据等 |
37+
* | 5 | 训练已经取消 |
38+
* | 6 | 未知状态，不清楚机器人训练情况，应该是存在其它BUG或运维事故导致，比如 redis 服务不可用 |
39+
*/
40+
// 意图识别训练状态
41+
private int retrain;
42+
// 支持库重新索引状态
43+
private int reindex;
44+
// 多轮对话重新解析状态
45+
private int reparse;
46+
47+
private String reparseMsg;
48+
private String retrainMsg;
49+
private String reindexMsg;
50+
51+
private String reindexUpdateAt;
52+
private String reparseUpdateAt;
53+
private String retrainUpdateAt;
54+
55+
public Status(final JSONObject json) throws ResourceInvalidException {
56+
this.retrain = json.optInt("retrain", -1);
57+
this.reindex = json.optInt("reindex", -1);
58+
this.reparse = json.optInt("reparse", -1);
59+
this.reindexUpdateAt = json.optString("reindexUpdateAt");
60+
this.reparseUpdateAt = json.optString("reparseUpdateAt");
61+
this.retrainUpdateAt = json.optString("retrainUpdateAt");
62+
this.reparseMsg = json.optString("reparseMsg");
63+
this.retrainMsg = json.optString("retrainMsg");
64+
this.reindexMsg = json.optString("reindexMsg");
65+
66+
if (!this.check()) {
67+
throw new ResourceInvalidException("Invalid data value");
68+
}
69+
}
70+
71+
/**
72+
* 检查数据合法性
73+
*
74+
* @return 是否有效
75+
*/
76+
private boolean check() {
77+
if (this.retrain == -1 || this.reindex == -1 || this.reparse == -1) {
78+
return false;
79+
}
80+
return true;
81+
}
82+
83+
84+
public int getRetrain() {
85+
return retrain;
86+
}
87+
88+
public void setRetrain(int retrain) {
89+
this.retrain = retrain;
90+
}
91+
92+
public int getReindex() {
93+
return reindex;
94+
}
95+
96+
public void setReindex(int reindex) {
97+
this.reindex = reindex;
98+
}
99+
100+
public int getReparse() {
101+
return reparse;
102+
}
103+
104+
public void setReparse(int reparse) {
105+
this.reparse = reparse;
106+
}
107+
108+
public String getReparseMsg() {
109+
return reparseMsg;
110+
}
111+
112+
public void setReparseMsg(String reparseMsg) {
113+
this.reparseMsg = reparseMsg;
114+
}
115+
116+
public String getRetrainMsg() {
117+
return retrainMsg;
118+
}
119+
120+
public void setRetrainMsg(String retrainMsg) {
121+
this.retrainMsg = retrainMsg;
122+
}
123+
124+
public String getReindexMsg() {
125+
return reindexMsg;
126+
}
127+
128+
public void setReindexMsg(String reindexMsg) {
129+
this.reindexMsg = reindexMsg;
130+
}
131+
132+
public String getReindexUpdateAt() {
133+
return reindexUpdateAt;
134+
}
135+
136+
public void setReindexUpdateAt(String reindexUpdateAt) {
137+
this.reindexUpdateAt = reindexUpdateAt;
138+
}
139+
140+
public String getReparseUpdateAt() {
141+
return reparseUpdateAt;
142+
}
143+
144+
public void setReparseUpdateAt(String reparseUpdateAt) {
145+
this.reparseUpdateAt = reparseUpdateAt;
146+
}
147+
148+
public String getRetrainUpdateAt() {
149+
return retrainUpdateAt;
150+
}
151+
152+
public void setRetrainUpdateAt(String retrainUpdateAt) {
153+
this.retrainUpdateAt = retrainUpdateAt;
154+
}
155+
156+
}

‎src/test/java/com/chatopera/bot/sdk/DictsMgrTest.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
*/
1616
package com.chatopera.bot.sdk;
1717

18-
import com.chatopera.bot.sdk.basics.Response;
1918
import com.chatopera.bot.exception.*;
19+
import com.chatopera.bot.sdk.basics.Response;
2020
import com.chatopera.bot.sdk.models.DictWord;
2121
import com.chatopera.bot.utils.EnvUtil;
2222
import com.chatopera.bot.utils.Logger;
@@ -25,8 +25,6 @@
2525
import org.json.JSONObject;
2626

2727
import java.io.IOException;
28-
import java.util.ArrayList;
29-
import java.util.List;
3028

3129
public class DictsMgrTest extends TestCase {
3230

0 commit comments

Comments
 (0)