Skip to content

Commit c09f3ca

Browse files
MeychiMeychi
committed
Progress
1 parent c80a257 commit c09f3ca

35 files changed

+9209
-1
lines changed

‎FlashDevelop/Managers/ScintillaManager.cs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,20 @@ public static void MakeFileWritable(ScintillaControl sci)
122122
/// </summary>
123123
public static Int32 SelectCodePage(Int32 codepage)
124124
{
125-
if (codepage == 65001 || codepage == 1201 || codepage == 1200) return 65001;
125+
if (codepage == 65001 /*UTF-8*/
126+
|| codepage == 1200 /*UTF-16LE*/
127+
|| codepage == 1201) /*UTF-16BE*/
128+
{
129+
return 65001;
130+
}
131+
else if (codepage == 932 /*Japanese Shift JIS*/
132+
|| codepage == 936 /*Simplified Chinese GBK*/
133+
|| codepage == 949 /*Korean Unified Hangul Code*/
134+
|| codepage == 950 /*Traditional Chinese Big5*/
135+
|| codepage == 1361 /*Korean Johab*/)
136+
{
137+
return codepage;
138+
}
126139
else return 0; // Disable multibyte support
127140
}
128141

‎PluginCore/PluginCore.csproj

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,39 @@
515515
<Compile Include="SharpZipLib\Zip\ZipInputStream.cs" />
516516
<Compile Include="SharpZipLib\Zip\ZipNameTransform.cs" />
517517
<Compile Include="SharpZipLib\Zip\ZipOutputStream.cs" />
518+
<Compile Include="UdeLibrary\Ude.Core\Big5Prober.cs" />
519+
<Compile Include="UdeLibrary\Ude.Core\BitPackage.cs" />
520+
<Compile Include="UdeLibrary\Ude.Core\CharDistributionAnalyser.cs" />
521+
<Compile Include="UdeLibrary\Ude.Core\CharsetProber.cs" />
522+
<Compile Include="UdeLibrary\Ude.Core\Charsets.cs" />
523+
<Compile Include="UdeLibrary\Ude.Core\CodingStateMachine.cs" />
524+
<Compile Include="UdeLibrary\Ude.Core\EscCharsetProber.cs" />
525+
<Compile Include="UdeLibrary\Ude.Core\EscSM.cs" />
526+
<Compile Include="UdeLibrary\Ude.Core\EUCJPProber.cs" />
527+
<Compile Include="UdeLibrary\Ude.Core\EUCKRProber.cs" />
528+
<Compile Include="UdeLibrary\Ude.Core\EUCTWProber.cs" />
529+
<Compile Include="UdeLibrary\Ude.Core\GB18030Prober.cs" />
530+
<Compile Include="UdeLibrary\Ude.Core\HebrewProber.cs" />
531+
<Compile Include="UdeLibrary\Ude.Core\JapaneseContextAnalyser.cs" />
532+
<Compile Include="UdeLibrary\Ude.Core\LangBulgarianModel.cs" />
533+
<Compile Include="UdeLibrary\Ude.Core\LangCyrillicModel.cs" />
534+
<Compile Include="UdeLibrary\Ude.Core\LangGreekModel.cs" />
535+
<Compile Include="UdeLibrary\Ude.Core\LangHebrewModel.cs" />
536+
<Compile Include="UdeLibrary\Ude.Core\LangHungarianModel.cs" />
537+
<Compile Include="UdeLibrary\Ude.Core\LangThaiModel.cs" />
538+
<Compile Include="UdeLibrary\Ude.Core\Latin1Prober.cs" />
539+
<Compile Include="UdeLibrary\Ude.Core\MBCSGroupProber.cs" />
540+
<Compile Include="UdeLibrary\Ude.Core\MBCSSM.cs" />
541+
<Compile Include="UdeLibrary\Ude.Core\SBCharsetProber.cs" />
542+
<Compile Include="UdeLibrary\Ude.Core\SBCSGroupProber.cs" />
543+
<Compile Include="UdeLibrary\Ude.Core\SequenceModel.cs" />
544+
<Compile Include="UdeLibrary\Ude.Core\SJISProber.cs" />
545+
<Compile Include="UdeLibrary\Ude.Core\SMModel.cs" />
546+
<Compile Include="UdeLibrary\Ude.Core\UniversalDetector.cs" />
547+
<Compile Include="UdeLibrary\Ude.Core\UTF8Prober.cs" />
548+
<Compile Include="UdeLibrary\Ude\CharsetDetector.cs" />
549+
<Compile Include="UdeLibrary\Ude\DetectionConfidence.cs" />
550+
<Compile Include="UdeLibrary\Ude\ICharsetDetector.cs" />
518551
</ItemGroup>
519552
<ItemGroup>
520553
<Service Include="{967B4E0D-AD0C-4609-AB67-0FA40C0206D8}" />

‎PluginCore/UdeLibrary/Ude.Core/Big5Prober.cs

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/* ***** BEGIN LICENSE BLOCK *****
2+
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
3+
*
4+
* The contents of this file are subject to the Mozilla Public License Version
5+
* 1.1 (the "License"); you may not use this file except in compliance with
6+
* the License. You may obtain a copy of the License at
7+
* http://www.mozilla.org/MPL/
8+
*
9+
* Software distributed under the License is distributed on an "AS IS" basis,
10+
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
11+
* for the specific language governing rights and limitations under the
12+
* License.
13+
*
14+
* The Original Code is Mozilla Universal charset detector code.
15+
*
16+
* The Initial Developer of the Original Code is
17+
* Netscape Communications Corporation.
18+
* Portions created by the Initial Developer are Copyright (C) 2001
19+
* the Initial Developer. All Rights Reserved.
20+
*
21+
* Contributor(s):
22+
* Shy Shalom <[email protected]>
23+
* Rudi Pettazzi <[email protected]> (C# port)
24+
*
25+
* Alternatively, the contents of this file may be used under the terms of
26+
* either the GNU General Public License Version 2 or later (the "GPL"), or
27+
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28+
* in which case the provisions of the GPL or the LGPL are applicable instead
29+
* of those above. If you wish to allow use of your version of this file only
30+
* under the terms of either the GPL or the LGPL, and not to allow others to
31+
* use your version of this file under the terms of the MPL, indicate your
32+
* decision by deleting the provisions above and replace them with the notice
33+
* and other provisions required by the GPL or the LGPL. If you do not delete
34+
* the provisions above, a recipient may use your version of this file under
35+
* the terms of any one of the MPL, the GPL or the LGPL.
36+
*
37+
* ***** END LICENSE BLOCK ***** */
38+
39+
using System;
40+
41+
namespace Ude.Core
42+
{
43+
public class Big5Prober : CharsetProber
44+
{
45+
//void GetDistribution(PRUint32 aCharLen, const char* aStr);
46+
private CodingStateMachine codingSM;
47+
private BIG5DistributionAnalyser distributionAnalyser;
48+
private byte[] lastChar = new byte[2];
49+
50+
public Big5Prober()
51+
{
52+
this.codingSM = new CodingStateMachine(new BIG5SMModel());
53+
this.distributionAnalyser = new BIG5DistributionAnalyser();
54+
this.Reset();
55+
}
56+
57+
public override ProbingState HandleData(byte[] buf, int offset, int len)
58+
{
59+
int codingState = 0;
60+
int max = offset + len;
61+
62+
for (int i = offset; i < max; i++) {
63+
codingState = codingSM.NextState(buf[i]);
64+
if (codingState == SMModel.ERROR) {
65+
state = ProbingState.NotMe;
66+
break;
67+
}
68+
if (codingState == SMModel.ITSME) {
69+
state = ProbingState.FoundIt;
70+
break;
71+
}
72+
if (codingState == SMModel.START) {
73+
int charLen = codingSM.CurrentCharLen;
74+
if (i == offset) {
75+
lastChar[1] = buf[offset];
76+
distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
77+
} else {
78+
distributionAnalyser.HandleOneChar(buf, i-1, charLen);
79+
}
80+
}
81+
}
82+
lastChar[0] = buf[max-1];
83+
84+
if (state == ProbingState.Detecting)
85+
if (distributionAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD)
86+
state = ProbingState.FoundIt;
87+
return state;
88+
}
89+
90+
public override void Reset()
91+
{
92+
codingSM.Reset();
93+
state = ProbingState.Detecting;
94+
distributionAnalyser.Reset();
95+
}
96+
97+
public override string GetCharsetName()
98+
{
99+
return "Big-5";
100+
}
101+
102+
public override float GetConfidence()
103+
{
104+
return distributionAnalyser.GetConfidence();
105+
}
106+
107+
}
108+
}

‎PluginCore/UdeLibrary/Ude.Core/BitPackage.cs

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/* ***** BEGIN LICENSE BLOCK *****
2+
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
3+
*
4+
* The contents of this file are subject to the Mozilla Public License Version
5+
* 1.1 (the "License"); you may not use this file except in compliance with
6+
* the License. You may obtain a copy of the License at
7+
* http://www.mozilla.org/MPL/
8+
*
9+
* Software distributed under the License is distributed on an "AS IS" basis,
10+
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
11+
* for the specific language governing rights and limitations under the
12+
* License.
13+
*
14+
* The Original Code is Mozilla Universal charset detector code.
15+
*
16+
* The Initial Developer of the Original Code is
17+
* Netscape Communications Corporation.
18+
* Portions created by the Initial Developer are Copyright (C) 2001
19+
* the Initial Developer. All Rights Reserved.
20+
*
21+
* Contributor(s):
22+
* Kohei TAKETA <[email protected]> (Java port)
23+
* Rudi Pettazzi <[email protected]> (C# port)
24+
*
25+
* Alternatively, the contents of this file may be used under the terms of
26+
* either the GNU General Public License Version 2 or later (the "GPL"), or
27+
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28+
* in which case the provisions of the GPL or the LGPL are applicable instead
29+
* of those above. If you wish to allow use of your version of this file only
30+
* under the terms of either the GPL or the LGPL, and not to allow others to
31+
* use your version of this file under the terms of the MPL, indicate your
32+
* decision by deleting the provisions above and replace them with the notice
33+
* and other provisions required by the GPL or the LGPL. If you do not delete
34+
* the provisions above, a recipient may use your version of this file under
35+
* the terms of any one of the MPL, the GPL or the LGPL.
36+
*
37+
* ***** END LICENSE BLOCK ***** */
38+
39+
using System;
40+
41+
namespace Ude.Core
42+
{
43+
public class BitPackage
44+
{
45+
public static int INDEX_SHIFT_4BITS = 3;
46+
public static int INDEX_SHIFT_8BITS = 2;
47+
public static int INDEX_SHIFT_16BITS = 1;
48+
49+
public static int SHIFT_MASK_4BITS = 7;
50+
public static int SHIFT_MASK_8BITS = 3;
51+
public static int SHIFT_MASK_16BITS = 1;
52+
53+
public static int BIT_SHIFT_4BITS = 2;
54+
public static int BIT_SHIFT_8BITS = 3;
55+
public static int BIT_SHIFT_16BITS = 4;
56+
57+
public static int UNIT_MASK_4BITS = 0x0000000F;
58+
public static int UNIT_MASK_8BITS = 0x000000FF;
59+
public static int UNIT_MASK_16BITS = 0x0000FFFF;
60+
61+
private int indexShift;
62+
private int shiftMask;
63+
private int bitShift;
64+
private int unitMask;
65+
private int[] data;
66+
67+
public BitPackage(int indexShift, int shiftMask,
68+
int bitShift, int unitMask, int[] data)
69+
{
70+
this.indexShift = indexShift;
71+
this.shiftMask = shiftMask;
72+
this.bitShift = bitShift;
73+
this.unitMask = unitMask;
74+
this.data = data;
75+
}
76+
77+
public static int Pack16bits(int a, int b)
78+
{
79+
return ((b << 16) | a);
80+
}
81+
82+
public static int Pack8bits(int a, int b, int c, int d)
83+
{
84+
return Pack16bits((b << 8) | a, (d << 8) | c);
85+
}
86+
87+
public static int Pack4bits(int a, int b, int c, int d,
88+
int e, int f, int g, int h)
89+
{
90+
return Pack8bits((b << 4) | a, (d << 4) | c,
91+
(f << 4) | e, (h << 4) | g);
92+
}
93+
94+
public int Unpack(int i)
95+
{
96+
return (data[i >> indexShift] >>
97+
((i & shiftMask) << bitShift)) & unitMask;
98+
}
99+
}
100+
}

0 commit comments

Comments
 (0)