Skip to content

Commit 90c0313

Browse files
MeychiMeychi
committed
Merge pull request #899 from fdorg/ude
Improved detection and handling of different 8-bit encodings, closes #875.
2 parents 4c0a767 + 31bda96 commit 90c0313

40 files changed

+9319
-69
lines changed

‎FlashDevelop/Bin/Debug/Settings/MainMenu.xml

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,21 +34,14 @@
3434
<button label="Label.CloseAll" click="CloseAll" shortcut="Control|Shift|V" flags="Enable:!IsAloneInPane" />
3535
<separator />
3636
<menu label="Label.Encoding" name="EncodingMenu" flags="Enable:IsEditable">
37-
<button label="Label.8Bits" click="ChangeEncoding" tag="0" flags="Check:IsEditable|IsActiveEncoding" />
37+
<button label="Label.8Bits" click="ChangeEncoding" tag="0" flags="Check:IsEditable|!IsUnicode" />
3838
<button label="Label.BigEndian" click="ChangeEncoding" tag="1201" flags="Check:IsEditable|IsActiveEncoding" />
3939
<button label="Label.LittleEndian" click="ChangeEncoding" tag="1200" flags="Check:IsEditable|IsActiveEncoding" />
4040
<button label="Label.UTF8" click="ChangeEncoding" tag="65001" flags="Check:IsEditable|IsActiveEncoding" />
4141
<button label="Label.UTF7" click="ChangeEncoding" tag="65000" flags="Check:IsEditable|IsActiveEncoding" />
4242
<separator />
4343
<button label="Label.SaveBOM" click="ToggleSaveBOM" flags="Check:IsEditable|SaveBOM+Disable:IsDefaultEncoding" />
4444
<separator />
45-
<menu label="Label.ConvertTextTo" name="ConvertToMenu" flags="Enable:IsEditable">
46-
<button label="Label.8Bits" click="ConvertEncoding" tag="0" image="559" />
47-
<button label="Label.BigEndian" click="ConvertEncoding" tag="1201" />
48-
<button label="Label.LittleEndian" click="ConvertEncoding" tag="1200" />
49-
<button label="Label.UTF8" click="ConvertEncoding" tag="65001" />
50-
<button label="Label.UTF7" click="ConvertEncoding" tag="65000" />
51-
</menu>
5245
<menu label="Label.OpenFilesIn" name="OpenInMenu" flags="Enable:IsEditable" image="559">
5346
<button label="Label.8Bits" click="OpenIn" tag="0" image="559" />
5447
<button label="Label.BigEndian" click="OpenIn" tag="1201" />

‎FlashDevelop/Docking/TabbedDocument.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,6 @@ public void Reload(Boolean showQuestion)
463463
Encoding encoding = Encoding.GetEncoding(info.CodePage);
464464
this.SciControl.IsReadOnly = false;
465465
this.SciControl.Encoding = encoding;
466-
this.SciControl.CodePage = ScintillaManager.SelectCodePage(info.CodePage);
467466
this.SciControl.Text = info.Contents;
468467
this.SciControl.IsReadOnly = FileHelper.FileIsReadOnly(this.FileName);
469468
this.SciControl.SetSel(position, position);

‎FlashDevelop/MainForm.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2223,6 +2223,11 @@ public void OpenIn(Object sender, System.EventArgs e)
22232223
Int32 count = this.openFileDialog.FileNames.Length;
22242224
for (Int32 i = 0; i < count; i++)
22252225
{
2226+
if (encMode == 0) // Detect 8bit encoding...
2227+
{
2228+
Int32 codepage = FileHelper.GetFileCodepage(openFileDialog.FileNames[i]);
2229+
encoding = Encoding.GetEncoding(codepage);
2230+
}
22262231
this.OpenEditableDocument(openFileDialog.FileNames[i], encoding, false);
22272232
}
22282233
}
@@ -2584,7 +2589,7 @@ public void Duplicate(Object sender, System.EventArgs e)
25842589
ScintillaControl sci = Globals.SciControl;
25852590
String extension = Path.GetExtension(sci.FileName);
25862591
String filename = DocumentManager.GetNewDocumentName(extension);
2587-
DockContent document = this.CreateEditableDocument(filename, sci.Text, sci.CodePage);
2592+
DockContent document = this.CreateEditableDocument(filename, sci.Text, sci.Encoding.CodePage);
25882593
((TabbedDocument)document).IsModified = true;
25892594
}
25902595

@@ -3149,7 +3154,6 @@ public void ChangeEncoding(Object sender, System.EventArgs e)
31493154
ToolStripItem button = (ToolStripItem)sender;
31503155
ScintillaControl sci = Globals.SciControl;
31513156
Int32 encMode = Convert.ToInt32(((ItemData)button.Tag).Tag);
3152-
sci.CodePage = ScintillaManager.SelectCodePage(encMode);
31533157
sci.Encoding = Encoding.GetEncoding(encMode);
31543158
this.OnScintillaControlUpdateControl(sci);
31553159
this.OnDocumentModify(this.CurrentDocument);
@@ -3185,10 +3189,9 @@ public void ConvertEncoding(Object sender, System.EventArgs e)
31853189
ToolStripItem button = (ToolStripItem)sender;
31863190
ScintillaControl sci = Globals.SciControl;
31873191
Int32 encMode = Convert.ToInt32(((ItemData)button.Tag).Tag);
3188-
Int32 curMode = sci.CodePage; // From current..
3192+
Int32 curMode = sci.Encoding.CodePage; // From current..
31893193
String converted = DataConverter.ChangeEncoding(sci.Text, curMode, encMode);
31903194
sci.Encoding = Encoding.GetEncoding(encMode);
3191-
sci.CodePage = ScintillaManager.SelectCodePage(encMode);
31923195
sci.Text = converted; // Set after codepage change
31933196
this.OnScintillaControlUpdateControl(sci);
31943197
this.OnDocumentModify(this.CurrentDocument);

‎FlashDevelop/Managers/ButtonManager.cs

Lines changed: 78 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
6969
{
7070
if (!document.IsUntitled) return false;
7171
}
72-
else if (action.Contains("HasBookmarks"))
72+
if (action.Contains("!HasBookmarks"))
7373
{
74-
if (!document.HasBookmarks) return false;
74+
if (document.HasBookmarks) return false;
7575
}
76-
else if (action.Contains("!HasBookmarks"))
76+
else if (action.Contains("HasBookmarks"))
7777
{
78-
if (document.HasBookmarks) return false;
78+
if (!document.HasBookmarks) return false;
7979
}
8080
if (action.Contains("!IsAloneInPane"))
8181
{
@@ -125,6 +125,19 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
125125
{
126126
if (!Globals.MainForm.MultiInstanceMode) return false;
127127
}
128+
if (action.Contains("!IsFullScreen"))
129+
{
130+
if (MainForm.Instance.IsFullScreen) return false;
131+
}
132+
else if (action.Contains("IsFullScreen"))
133+
{
134+
if (!MainForm.Instance.IsFullScreen) return false;
135+
}
136+
if (action.Contains("TracksBoolean"))
137+
{
138+
Boolean value = (Boolean)Globals.Settings.GetValue(((ItemData)item.Tag).Tag);
139+
if (!value) return false;
140+
}
128141
if (sci != null)
129142
{
130143
if (action.Contains("!CanUndo"))
@@ -159,6 +172,31 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
159172
{
160173
if (sci.SelText.Length == 0) return false;
161174
}
175+
if (action.Contains("!SaveBOM"))
176+
{
177+
if (document.SciControl.SaveBOM) return false;
178+
}
179+
else if (action.Contains("SaveBOM"))
180+
{
181+
if (!document.SciControl.SaveBOM) return false;
182+
}
183+
if (action.Contains("!IsUnicode"))
184+
{
185+
if (ScintillaManager.IsUnicode(document.SciControl.Encoding.CodePage)) return false;
186+
}
187+
else if (action.Contains("IsUnicode"))
188+
{
189+
if (!ScintillaManager.IsUnicode(document.SciControl.Encoding.CodePage)) return false;
190+
}
191+
if (action.Contains("SyntaxIs?"))
192+
{
193+
String[] chunks = action.Split('?');
194+
if (chunks.Length == 2)
195+
{
196+
String language = document.SciControl.ConfigurationLanguage;
197+
if (chunks[chunks.Length - 1] != language.ToUpper()) return false;
198+
}
199+
}
162200
if (action.Contains("IsActiveSyntax"))
163201
{
164202
String language = document.SciControl.ConfigurationLanguage;
@@ -170,43 +208,17 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
170208
if (codepage == Encoding.Default.CodePage) codepage = 0;
171209
if (((ItemData)item.Tag).Tag != codepage.ToString()) return false;
172210
}
173-
if (action.Contains("SaveBOM"))
174-
{
175-
return document.SciControl.SaveBOM;
176-
}
177-
if (action.Contains("IsDefaultEncoding"))
178-
{
179-
Int32 codepage = document.SciControl.Encoding.CodePage;
180-
return codepage == Encoding.Default.CodePage;
181-
}
182211
if (action.Contains("IsActiveEOL"))
183212
{
184213
Int32 eolMode = document.SciControl.EOLMode;
185214
if (((ItemData)item.Tag).Tag != eolMode.ToString()) return false;
186215
}
187-
if (action.Contains("SyntaxIs?"))
216+
if (action.Contains("IsDefaultEncoding"))
188217
{
189-
String[] chunks = action.Split('?');
190-
if (chunks.Length == 2)
191-
{
192-
String language = document.SciControl.ConfigurationLanguage;
193-
if (chunks[chunks.Length - 1] != language.ToUpper()) return false;
194-
}
218+
Int32 codepage = document.SciControl.Encoding.CodePage;
219+
if (codepage != Encoding.Default.CodePage) return false;
195220
}
196221
}
197-
if (action.Contains("!IsFullScreen"))
198-
{
199-
if (MainForm.Instance.IsFullScreen) return false;
200-
}
201-
else if (action.Contains("IsFullScreen"))
202-
{
203-
if (!MainForm.Instance.IsFullScreen) return false;
204-
}
205-
if (action.Contains("TracksBoolean"))
206-
{
207-
Boolean value = (Boolean)Globals.Settings.GetValue(((ItemData)item.Tag).Tag);
208-
if (!value) return false;
209-
}
210222
return true;
211223
}
212224

@@ -302,35 +314,53 @@ public static String GetActiveEncodingName()
302314
ITabbedDocument document = Globals.CurrentDocument;
303315
if (document != null && document.IsEditable)
304316
{
305-
Boolean hasBOM = document.SciControl.SaveBOM;
306317
Int32 codepage = document.SciControl.Encoding.CodePage;
307-
if (codepage == Encoding.UTF8.CodePage)
308-
{
309-
return GetLabelAsPlainText("Label.UTF8", hasBOM);
310-
}
311-
else if (codepage == Encoding.UTF7.CodePage)
312-
{
313-
return GetLabelAsPlainText("Label.UTF7", hasBOM);
314-
}
315-
else if (codepage == Encoding.BigEndianUnicode.CodePage)
318+
EncodingFileInfo info = FileHelper.GetEncodingFileInfo(document.FileName);
319+
if (codepage == info.CodePage)
316320
{
317-
return GetLabelAsPlainText("Label.BigEndian", hasBOM);
321+
if (ScintillaManager.IsUnicode(info.CodePage))
322+
{
323+
String name = "Unicode (" + info.Charset + ")";
324+
return info.ContainsBOM ? name + " (BOM)" : name;
325+
}
326+
else
327+
{
328+
String name = TextHelper.GetString("Label.8Bits").Replace("&", "");
329+
return name + " (" + info.Charset + ")";
330+
}
318331
}
319-
else if (codepage == Encoding.Unicode.CodePage)
332+
else // Opened in different encoding...
320333
{
321-
return GetLabelAsPlainText("Label.LittleEndian", hasBOM);
334+
Boolean hasBOM = document.SciControl.SaveBOM;
335+
if (codepage == Encoding.UTF8.CodePage)
336+
{
337+
return GetLabelAsPlainText("Label.UTF8", true, hasBOM);
338+
}
339+
else if (codepage == Encoding.UTF7.CodePage)
340+
{
341+
return GetLabelAsPlainText("Label.UTF7", true, hasBOM);
342+
}
343+
else if (codepage == Encoding.BigEndianUnicode.CodePage)
344+
{
345+
return GetLabelAsPlainText("Label.BigEndian", true, hasBOM);
346+
}
347+
else if (codepage == Encoding.Unicode.CodePage)
348+
{
349+
return GetLabelAsPlainText("Label.LittleEndian", true, hasBOM);
350+
}
351+
else return GetLabelAsPlainText("Label.8Bits", false, false);
322352
}
323-
else return GetLabelAsPlainText("Label.8Bits", false);
324353
}
325354
else return TextHelper.GetString("Info.Unknown");
326355
}
327356

328357
/// <summary>
329358
/// Gets a label as plain text by removing the accelerator key
330359
/// </summary>
331-
public static String GetLabelAsPlainText(String name, Boolean hasBOM)
360+
public static String GetLabelAsPlainText(String name, Boolean unicode, Boolean hasBOM)
332361
{
333362
String label = TextHelper.GetString(name).Replace("&", "");
363+
if (unicode) label = "Unicode (" + label.ToLower() + ")";
334364
return hasBOM ? label + " (BOM)" : label;
335365
}
336366

‎FlashDevelop/Managers/ScintillaManager.cs

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,15 @@ public static void MakeFileWritable(ScintillaControl sci)
118118
}
119119

120120
/// <summary>
121-
/// Selects a correct codepage for the editor
121+
/// Detects from codepage if the encoding is unicode
122122
/// </summary>
123-
public static Int32 SelectCodePage(Int32 codepage)
123+
public static Boolean IsUnicode(Int32 codepage)
124124
{
125-
if (codepage == 65001 || codepage == 1201 || codepage == 1200) return 65001;
126-
else return 0; // Disable multibyte support
125+
return (codepage == Encoding.UTF7.CodePage
126+
|| codepage == Encoding.UTF8.CodePage
127+
|| codepage == Encoding.UTF32.CodePage
128+
|| codepage == Encoding.BigEndianUnicode.CodePage
129+
|| codepage == Encoding.Unicode.CodePage);
127130
}
128131

129132
/// <summary>
@@ -371,9 +374,9 @@ public static ScintillaControl CreateControl(String file, String text, Int32 cod
371374
sci.SetXCaretPolicy((Int32)(CaretPolicy.Jumps | CaretPolicy.Even), 30);
372375
sci.SetYCaretPolicy((Int32)(CaretPolicy.Jumps | CaretPolicy.Even), 2);
373376
sci.ScrollWidthTracking = (Globals.Settings.ScrollWidth == 3000);
374-
sci.CodePage = SelectCodePage(codepage);
377+
sci.CodePage = 65001; // Editor handles text as UTF-8
375378
sci.Encoding = Encoding.GetEncoding(codepage);
376-
sci.SaveBOM = (sci.CodePage == 65001) && Globals.Settings.SaveUnicodeWithBOM;
379+
sci.SaveBOM = IsUnicode(codepage) && Globals.Settings.SaveUnicodeWithBOM;
377380
sci.Text = text; sci.FileName = file; // Set text and save file name
378381
sci.Modified += new ModifiedHandler(Globals.MainForm.OnScintillaControlModified);
379382
sci.MarginClick += new MarginClickHandler(Globals.MainForm.OnScintillaControlMarginClick);

‎PluginCore/PluginCore.csproj

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,39 @@
515515
<Compile Include="SharpZipLib\Zip\ZipInputStream.cs" />
516516
<Compile Include="SharpZipLib\Zip\ZipNameTransform.cs" />
517517
<Compile Include="SharpZipLib\Zip\ZipOutputStream.cs" />
518+
<Compile Include="UdeLibrary\Ude.Core\Big5Prober.cs" />
519+
<Compile Include="UdeLibrary\Ude.Core\BitPackage.cs" />
520+
<Compile Include="UdeLibrary\Ude.Core\CharDistributionAnalyser.cs" />
521+
<Compile Include="UdeLibrary\Ude.Core\CharsetProber.cs" />
522+
<Compile Include="UdeLibrary\Ude.Core\Charsets.cs" />
523+
<Compile Include="UdeLibrary\Ude.Core\CodingStateMachine.cs" />
524+
<Compile Include="UdeLibrary\Ude.Core\EscCharsetProber.cs" />
525+
<Compile Include="UdeLibrary\Ude.Core\EscSM.cs" />
526+
<Compile Include="UdeLibrary\Ude.Core\EUCJPProber.cs" />
527+
<Compile Include="UdeLibrary\Ude.Core\EUCKRProber.cs" />
528+
<Compile Include="UdeLibrary\Ude.Core\EUCTWProber.cs" />
529+
<Compile Include="UdeLibrary\Ude.Core\GB18030Prober.cs" />
530+
<Compile Include="UdeLibrary\Ude.Core\HebrewProber.cs" />
531+
<Compile Include="UdeLibrary\Ude.Core\JapaneseContextAnalyser.cs" />
532+
<Compile Include="UdeLibrary\Ude.Core\LangBulgarianModel.cs" />
533+
<Compile Include="UdeLibrary\Ude.Core\LangCyrillicModel.cs" />
534+
<Compile Include="UdeLibrary\Ude.Core\LangGreekModel.cs" />
535+
<Compile Include="UdeLibrary\Ude.Core\LangHebrewModel.cs" />
536+
<Compile Include="UdeLibrary\Ude.Core\LangHungarianModel.cs" />
537+
<Compile Include="UdeLibrary\Ude.Core\LangThaiModel.cs" />
538+
<Compile Include="UdeLibrary\Ude.Core\Latin1Prober.cs" />
539+
<Compile Include="UdeLibrary\Ude.Core\MBCSGroupProber.cs" />
540+
<Compile Include="UdeLibrary\Ude.Core\MBCSSM.cs" />
541+
<Compile Include="UdeLibrary\Ude.Core\SBCharsetProber.cs" />
542+
<Compile Include="UdeLibrary\Ude.Core\SBCSGroupProber.cs" />
543+
<Compile Include="UdeLibrary\Ude.Core\SequenceModel.cs" />
544+
<Compile Include="UdeLibrary\Ude.Core\SJISProber.cs" />
545+
<Compile Include="UdeLibrary\Ude.Core\SMModel.cs" />
546+
<Compile Include="UdeLibrary\Ude.Core\UniversalDetector.cs" />
547+
<Compile Include="UdeLibrary\Ude.Core\UTF8Prober.cs" />
548+
<Compile Include="UdeLibrary\Ude\CharsetDetector.cs" />
549+
<Compile Include="UdeLibrary\Ude\DetectionConfidence.cs" />
550+
<Compile Include="UdeLibrary\Ude\ICharsetDetector.cs" />
518551
</ItemGroup>
519552
<ItemGroup>
520553
<Service Include="{967B4E0D-AD0C-4609-AB67-0FA40C0206D8}" />

0 commit comments

Comments
 (0)