【概述】做好一个web系统的安全运维，除了常规的防注入，防入侵等，还有一个检测并过滤敏感词，脏词.. 这件事做得不好，轻则导致一场投诉或纠纷，重则导致产品被勒令关闭停运。
废话少说，先看下代码,可以拿过去直接使用。
我要评论 1 using microsoft.visualbasic;
2 using system;
3 using system.collections.generic;
4 using system.io;
5 using system.linq;
6 using system.text;
7
8 namespace opencore.contentsecurity
9 {
10 /// <summary>
11 /// 功能简介：基于dfa算法的高效率非法关键词检测过滤类(杜绝违法内容)
12 /// 开发前参考内容：https://blog.csdn.net/u011966339/article/details/72832197
13 /// 更新日志：
14 /// 2020-4-15：加载字典的处理采用静态构造方法中处理，避免频繁加载，提升性能.
15 /// 支持多词库文件加载.
16 /// 优化了算法的细节，提高健壮性。
17 /// </summary>
18 public class sensitivewordfilter
19 {
20 private static string[] dictionarypathlist = null;
21 /// <summary>
22 /// 内存词典
23 /// </summary>
24 private static wordgroup[] memorylexicon = new wordgroup[(int)char.maxvalue];
25 private static object lockobj = new object();
26 public static void init(string[] sdictionaryfilename)
27 {
28 dictionarypathlist = sdictionaryfilename;
29 loaddictionary();
30 }
31 public sensitivewordfilter()
32 {
33
34 }
35 private string sourcttext = string.empty;
36 /// <summary>
37 /// 检测源
38 /// </summary>
39 private string sourcttext
40 {
41 get { return sourcttext; }
42 set { sourcttext = value; }
43 }
44 /// <summary>
45 /// 检测源游标
46 /// </summary>
47 private int cursor = 0;
48 /// <summary>
49 /// 匹配成功后偏移量
50 /// </summary>
51 private int wordlenght = 0;
52 /// <summary>
53 /// 检测词游标
54 /// </summary>
55 private int nextcursor = 0;
56 private list<string> illegalwords = new list<string>();
57 /// <summary>
58 /// 检测到的非法词集
59 /// </summary>
60 public list<string> illegalwords
61 {
62 get { return illegalwords; }
63 }
64 /// <summary>
65 /// 判断是否是中文
66 /// </summary>
67 /// <param name="character"></param>
68 /// <returns></returns>
69 private bool ischs(char character)
70 {
71 // 中文表意字符的范围 4e00-9fa5
72 int charval = (int)character;
73 return (charval >= 0x4e00 && charval <= 0x9fa5);
74 }
75 /// <summary>
76 /// 判断是否是数字
77 /// </summary>
78 /// <param name="character"></param>
79 /// <returns></returns>
80 private bool isnum(char character)
81 {
82 int charval = (int)character;
83 return (charval >= 48 && charval <= 57);
84 }
85 /// <summary>
86 /// 判断是否是字母
87 /// </summary>
88 /// <param name="character"></param>
89 /// <returns></returns>
90 private bool isalphabet(char character)
91 {
92 int charval = (int)character;
93 return ((charval >= 97 && charval <= 122) || (charval >= 65 && charval <= 90));
94 }
95 /// <summary>
96 /// 转半角小写的函数(dbc case)
97 /// </summary>
98 /// <param name="input">任意字符串</param>
99 /// <returns>半角字符串</returns>
100 ///<remarks>
101 ///全角空格为12288，半角空格为32
102 ///其他字符半角(33-126)与全角(65281-65374)的对应关系是：均相差65248
103 ///</remarks>
104 private static string todbc(string input)
105 {
106 char[] c = input.tochararray();
107 for (int i = 0; i < c.length; i++)
108 {
109 if (c[i] == 12288)
110 {
111 c[i] = (char)32;
112 continue;
113 }
114 if (c[i] > 65280 && c[i] < 65375)
115 c[i] = (char)(c[i] - 65248);
116 }
117 return new string(c).tolower();
118 }
119 /// <summary>
120 /// 转换为简体中文
121 /// </summary>
122 /// <param name="sinput"></param>
123 /// <returns></returns>
124 private static string tosimplifiedchiniese(string sinput)
125 {
126 if (string.isnullorempty(sinput))
127 {
128 return string.empty;
129 }
130 try
131 {
132 return strings.strconv(sinput, vbstrconv.simplifiedchinese, 0);
133 }
134 catch (exception ex)
135 {
136
137 }
138 return sinput;
139 }
140 /// <summary>
141 /// 写入日志(非跨程序域的场景)
142 /// </summary>
143 /// <param name="msg"></param>
144 private static void savelog(string msg)
145 {
146 string spath = path.combine(appdomain.currentdomain.setupinformation.applicationbase, "securitylog");
147 if (!directory.exists(spath))
148 {
149 directory.createdirectory(spath);
150 }
151 spath = string.format("{0}\\{1}", spath, datetime.now.tostring("yyyymmdd") + ".log");
152 try
153 {
154 file.appendalltext(spath, "[" + datetime.now.tostring() + "]" + msg + "\r\n");
155 }
156 catch
157 {
158 }
159 }
160 /// <summary>
161 /// 加载内存词库
162 /// </summary>
163 private static void loaddictionary()
164 {
165 if (dictionarypathlist == null || dictionarypathlist.length == 0)
166 {
167 savelog($"sensitivewordfilter.loaddictionary.字典路径配置为空");
168 return;
169 }
170 foreach (string sfilename in dictionarypathlist)
171 {
172 if (file.exists(sfilename) == false)
173 {
174 savelog($"sensitivewordfilter.loaddictionary.路径:{sfilename}不是一个有效的文件");
175 return;
176 }
177 }
178 list<string> wordlist = new list<string>();
179 array.clear(memorylexicon, 0, memorylexicon.length);
180 foreach (string sdictionaryfile in dictionarypathlist)
181 {
182 string[] words = system.io.file.readalllines(sdictionaryfile, system.text.encoding.default);
183 foreach (string word in words)
184 {
185 if (string.isnullorempty(word))
186 continue;
187 if (word.trim().length == 0)
188 continue;
189 string key = todbc(word);
190 wordlist.add(key);
191 //适配繁体，简体.addbyww@2020-4-15
192 string key_simple = tosimplifiedchiniese(key);
193 if (key_simple != key)
194 {
195 wordlist.add(key_simple);
196 }
197 }
198 }
199 comparison<string> cmp = delegate (string key1, string key2)
200 {
201 return key1.compareto(key2);
202 };
203 wordlist.sort(cmp);
204 for (int i = wordlist.count - 1; i > 0; i--)
205 {
206 if (wordlist[i].tostring() == wordlist[i - 1].tostring())
207 {
208 wordlist.removeat(i);
209 }
210 }
211 foreach (var word in wordlist)
212 {
213 if (word.length > 0)
214 {
215 wordgroup group = memorylexicon[(int)word[0]];
216 if (group == null)
217 {
218 group = new wordgroup();
219 memorylexicon[(int)word[0]] = group;
220 }
221 group.add(word.substring(1));
222 }
223 }
224 }
225 /// <summary>
226 /// 检测
227 /// </summary>
228 /// <param name="blackword"></param>
229 /// <returns></returns>
230 private bool check(string blackword)
231 {
232 wordlenght = 0;
233 //检测源下一位游标
234 nextcursor = cursor + 1;
235 bool found = false;
236 //遍历词的每一位做匹配
237 for (int i = 0; i < blackword.length; i++)
238 {
239 //特殊字符偏移游标
240 int offset = 0;
241 if (nextcursor >= sourcttext.length)
242 {
243 break;
244 }
245 else
246 {
247 //检测下位字符如果不是汉字 数字 字符 偏移量加1
248 for (int y = nextcursor; y < sourcttext.length; y++)
249 {
250
251 if (!ischs(sourcttext[y]) && !isnum(sourcttext[y]) && !isalphabet(sourcttext[y]))
252 {
253 offset++;
254 //避让特殊字符，下位游标如果>=字符串长度 跳出
255 if (nextcursor + offset >= sourcttext.length) break;
256 wordlenght++;
257 }
258 else break;
259 }
260 if ((int)blackword[i] == (int)sourcttext[nextcursor + offset])
261 {
262 found = true;
263 }
264 else
265 {
266 found = false;
267 break;
268 }
269 }
270 nextcursor = nextcursor + 1 + offset;
271 wordlenght++;
272 }
273 return found;
274 }
275 /// <summary>
276 /// 检测并替换敏感词为指定字符。之后返回
277 /// </summary>
278 /// <param name="replacechar">比如：*</param>
279 public string getdatabyfilter(string ssourceinput, char replacechar)
280 {
281 if (string.isnullorempty(ssourceinput))
282 {
283 return ssourceinput;
284 }
285 if (memorylexicon == null || memorylexicon.length == 0)
286 {
287 savelog($"sensitivewordfilter.getdatabyfilter.内存字典为空");
288 return ssourceinput;
289 }
290 //初始化
291 this.cursor = 0;
292 this.wordlenght = 0;
293 this.illegalwords.clear();
294 this.sourcttext = ssourceinput;
295 if (sourcttext != string.empty)
296 {
297 char[] tempstring = sourcttext.tochararray();
298 for (int i = 0; i < sourcttext.length; i++)
299 {
300 //查询以该字为首字符的词组
301 wordgroup group = memorylexicon[(int)todbc(sourcttext)[i]];
302 if (group != null)
303 {
304 for (int z = 0; z < group.count(); z++)
305 {
306 string word = group.getword(z);
307 if (word.length == 0 || check(word))
308 {
309 string blackword = string.empty;
310 for (int pos = 0; pos < wordlenght + 1; pos++)
311 {
312 blackword += tempstring[pos + cursor].tostring();
313 tempstring[pos + cursor] = replacechar;
314 }
315 illegalwords.add(blackword);
316 cursor = cursor + wordlenght;
317 i = i + wordlenght;
318 }
319 }
320 }
321 cursor++;
322 }
323 return new string(tempstring);
324 }
325 else
326 {
327 return string.empty;
328 }
329 }
330 }
331 /// <summary>
332 /// 具有相同首字符的词组集合
333 /// </summary>
334 public class wordgroup
335 {
336 /// <summary>
337 /// 集合
338 /// </summary>
339 private list<string> grouplist=new list<string>();
340 public wordgroup()
341 {
342
343 }
344 /// <summary>
345 /// 添加词
346 /// </summary>
347 /// <param name="word"></param>
348 public void add(string word)
349 {
350 if (grouplist.contains(word) == false)
351 {
352 grouplist.add(word);
353 }
354 }
355 /// <summary>
356 /// 获取总数
357 /// </summary>
358 /// <returns></returns>
359 public int count()
360 {
361 return grouplist.count;
362 }
363 /// <summary>
364 /// 根据下标获取词
365 /// </summary>
366 /// <param name="index"></param>
367 /// <returns></returns>
368 public string getword(int index)
369 {
370 return grouplist[index];
371 }
372 }
373 }
上面是一个完整的，独立的实现类。 下面给一个简单的调用示例:
1 //全局配置,整个程序只要配置一次即可，后续无需配置
2 sensitivewordfilter.init(new string[] {
3 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\暴恐词库.txt",
4 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\反动词库.txt",
5 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\民生词库.txt",
6 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\色情词库.txt",
7 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\贪腐词库.txt",
8 @"c:\users\x\downloads\网站需要过滤的敏感词\mgck-master\其他词库.txt"
9 });
10 //下列可以在多个地方实例化，可以并发执行
11 sensitivewordfilter wordfilter = new sensitivewordfilter();
12 dictionary<string, string> dicttestdata = new dictionary<string, string>();
13 //多测几个示例，看看效果
14 dicttestdata["杀^人游戏,有人找一夜q"] = "";
15 dicttestdata["数学学习课堂"] = "";
16 dicttestdata["打击法0功有，法0功毒害大众"] = "";
17 dictionary<string, string> dictresult = new dictionary<string, string>();
18 foreach(string skey in dicttestdata.keys)
19 {
20 dictresult[skey] = $"替换后:{wordfilter.getdatabyfilter(skey,'|')}, ------------检测违禁词:{string.join(",",(wordfilter.illegalwords==null?new list<string>():wordfilter.illegalwords))}";
21 }
22 string sresultjson = jsonconverter.serializeobject(dictresult);
23 utils.savelog(sresultjson);
最后，给一下打印的结果：
"杀^人游戏,有人找一夜q": 替换后: "杀^人游戏,有人找|||", ------------检测违禁词:一夜q",
"数学学习课堂": 替换后:"数学学习课堂", ------------检测违禁词:,
"打击法0功有，法0功毒害大众": 替换后:"打击|||有，|||毒害大众", ------------检测违禁词:法0功,法0功"
-------------附
词库下载地址：https://codeload.github.com/chason777777/mgck/zip/master
如对本文有疑问， 点击进行留言回复！！
使用Visual Studio2019创建C#项目(窗体应用程序、控制台应用程序、Web应用程序)
C#实现获取本地内网(局域网)和外网(公网)IP地址的方法分析
网友评论