Skip to content

Commit dce957d

Browse files
MeychiMeychi
committed
Finishing display and charset detection functionality...
1 parent c09f3ca commit dce957d

File tree

7 files changed

+123
-82
lines changed

7 files changed

+123
-82
lines changed

‎FlashDevelop/Bin/Debug/Settings/MainMenu.xml

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,21 +34,14 @@
3434
<button label="Label.CloseAll" click="CloseAll" shortcut="Control|Shift|V" flags="Enable:!IsAloneInPane" />
3535
<separator />
3636
<menu label="Label.Encoding" name="EncodingMenu" flags="Enable:IsEditable">
37-
<button label="Label.8Bits" click="ChangeEncoding" tag="0" flags="Check:IsEditable|IsActiveEncoding" />
37+
<button label="Label.8Bits" click="ChangeEncoding" tag="0" flags="Check:IsEditable|!IsUnicode" />
3838
<button label="Label.BigEndian" click="ChangeEncoding" tag="1201" flags="Check:IsEditable|IsActiveEncoding" />
3939
<button label="Label.LittleEndian" click="ChangeEncoding" tag="1200" flags="Check:IsEditable|IsActiveEncoding" />
4040
<button label="Label.UTF8" click="ChangeEncoding" tag="65001" flags="Check:IsEditable|IsActiveEncoding" />
4141
<button label="Label.UTF7" click="ChangeEncoding" tag="65000" flags="Check:IsEditable|IsActiveEncoding" />
4242
<separator />
4343
<button label="Label.SaveBOM" click="ToggleSaveBOM" flags="Check:IsEditable|SaveBOM+Disable:IsDefaultEncoding" />
4444
<separator />
45-
<menu label="Label.ConvertTextTo" name="ConvertToMenu" flags="Enable:IsEditable">
46-
<button label="Label.8Bits" click="ConvertEncoding" tag="0" image="559" />
47-
<button label="Label.BigEndian" click="ConvertEncoding" tag="1201" />
48-
<button label="Label.LittleEndian" click="ConvertEncoding" tag="1200" />
49-
<button label="Label.UTF8" click="ConvertEncoding" tag="65001" />
50-
<button label="Label.UTF7" click="ConvertEncoding" tag="65000" />
51-
</menu>
5245
<menu label="Label.OpenFilesIn" name="OpenInMenu" flags="Enable:IsEditable" image="559">
5346
<button label="Label.8Bits" click="OpenIn" tag="0" image="559" />
5447
<button label="Label.BigEndian" click="OpenIn" tag="1201" />

‎FlashDevelop/Docking/TabbedDocument.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,6 @@ public void Reload(Boolean showQuestion)
463463
Encoding encoding = Encoding.GetEncoding(info.CodePage);
464464
this.SciControl.IsReadOnly = false;
465465
this.SciControl.Encoding = encoding;
466-
this.SciControl.CodePage = ScintillaManager.SelectCodePage(info.CodePage);
467466
this.SciControl.Text = info.Contents;
468467
this.SciControl.IsReadOnly = FileHelper.FileIsReadOnly(this.FileName);
469468
this.SciControl.SetSel(position, position);

‎FlashDevelop/MainForm.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2223,6 +2223,11 @@ public void OpenIn(Object sender, System.EventArgs e)
22232223
Int32 count = this.openFileDialog.FileNames.Length;
22242224
for (Int32 i = 0; i < count; i++)
22252225
{
2226+
if (encMode == 0) // Detect 8bit encoding...
2227+
{
2228+
Int32 codepage = FileHelper.GetFileCodepage(openFileDialog.FileNames[i]);
2229+
encoding = Encoding.GetEncoding(codepage);
2230+
}
22262231
this.OpenEditableDocument(openFileDialog.FileNames[i], encoding, false);
22272232
}
22282233
}
@@ -2584,7 +2589,7 @@ public void Duplicate(Object sender, System.EventArgs e)
25842589
ScintillaControl sci = Globals.SciControl;
25852590
String extension = Path.GetExtension(sci.FileName);
25862591
String filename = DocumentManager.GetNewDocumentName(extension);
2587-
DockContent document = this.CreateEditableDocument(filename, sci.Text, sci.CodePage);
2592+
DockContent document = this.CreateEditableDocument(filename, sci.Text, sci.Encoding.CodePage);
25882593
((TabbedDocument)document).IsModified = true;
25892594
}
25902595

@@ -3149,7 +3154,6 @@ public void ChangeEncoding(Object sender, System.EventArgs e)
31493154
ToolStripItem button = (ToolStripItem)sender;
31503155
ScintillaControl sci = Globals.SciControl;
31513156
Int32 encMode = Convert.ToInt32(((ItemData)button.Tag).Tag);
3152-
sci.CodePage = ScintillaManager.SelectCodePage(encMode);
31533157
sci.Encoding = Encoding.GetEncoding(encMode);
31543158
this.OnScintillaControlUpdateControl(sci);
31553159
this.OnDocumentModify(this.CurrentDocument);
@@ -3185,10 +3189,9 @@ public void ConvertEncoding(Object sender, System.EventArgs e)
31853189
ToolStripItem button = (ToolStripItem)sender;
31863190
ScintillaControl sci = Globals.SciControl;
31873191
Int32 encMode = Convert.ToInt32(((ItemData)button.Tag).Tag);
3188-
Int32 curMode = sci.CodePage; // From current..
3192+
Int32 curMode = sci.Encoding.CodePage; // From current..
31893193
String converted = DataConverter.ChangeEncoding(sci.Text, curMode, encMode);
31903194
sci.Encoding = Encoding.GetEncoding(encMode);
3191-
sci.CodePage = ScintillaManager.SelectCodePage(encMode);
31923195
sci.Text = converted; // Set after codepage change
31933196
this.OnScintillaControlUpdateControl(sci);
31943197
this.OnDocumentModify(this.CurrentDocument);

‎FlashDevelop/Managers/ButtonManager.cs

Lines changed: 76 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
6969
{
7070
if (!document.IsUntitled) return false;
7171
}
72-
else if (action.Contains("HasBookmarks"))
72+
if (action.Contains("!HasBookmarks"))
7373
{
74-
if (!document.HasBookmarks) return false;
74+
if (document.HasBookmarks) return false;
7575
}
76-
else if (action.Contains("!HasBookmarks"))
76+
else if (action.Contains("HasBookmarks"))
7777
{
78-
if (document.HasBookmarks) return false;
78+
if (!document.HasBookmarks) return false;
7979
}
8080
if (action.Contains("!IsAloneInPane"))
8181
{
@@ -125,6 +125,19 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
125125
{
126126
if (!Globals.MainForm.MultiInstanceMode) return false;
127127
}
128+
if (action.Contains("!IsFullScreen"))
129+
{
130+
if (MainForm.Instance.IsFullScreen) return false;
131+
}
132+
else if (action.Contains("IsFullScreen"))
133+
{
134+
if (!MainForm.Instance.IsFullScreen) return false;
135+
}
136+
if (action.Contains("TracksBoolean"))
137+
{
138+
Boolean value = (Boolean)Globals.Settings.GetValue(((ItemData)item.Tag).Tag);
139+
if (!value) return false;
140+
}
128141
if (sci != null)
129142
{
130143
if (action.Contains("!CanUndo"))
@@ -159,6 +172,31 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
159172
{
160173
if (sci.SelText.Length == 0) return false;
161174
}
175+
if (action.Contains("!SaveBOM"))
176+
{
177+
if (document.SciControl.SaveBOM) return false;
178+
}
179+
else if (action.Contains("SaveBOM"))
180+
{
181+
if (!document.SciControl.SaveBOM) return false;
182+
}
183+
if (action.Contains("!IsUnicode"))
184+
{
185+
if (ScintillaManager.IsUnicode(document.SciControl.Encoding.CodePage)) return false;
186+
}
187+
else if (action.Contains("IsUnicode"))
188+
{
189+
if (!ScintillaManager.IsUnicode(document.SciControl.Encoding.CodePage)) return false;
190+
}
191+
if (action.Contains("SyntaxIs?"))
192+
{
193+
String[] chunks = action.Split('?');
194+
if (chunks.Length == 2)
195+
{
196+
String language = document.SciControl.ConfigurationLanguage;
197+
if (chunks[chunks.Length - 1] != language.ToUpper()) return false;
198+
}
199+
}
162200
if (action.Contains("IsActiveSyntax"))
163201
{
164202
String language = document.SciControl.ConfigurationLanguage;
@@ -170,43 +208,17 @@ public static Boolean ValidateFlagAction(ToolStripItem item, String action)
170208
if (codepage == Encoding.Default.CodePage) codepage = 0;
171209
if (((ItemData)item.Tag).Tag != codepage.ToString()) return false;
172210
}
173-
if (action.Contains("SaveBOM"))
174-
{
175-
return document.SciControl.SaveBOM;
176-
}
177-
if (action.Contains("IsDefaultEncoding"))
178-
{
179-
Int32 codepage = document.SciControl.Encoding.CodePage;
180-
return codepage == Encoding.Default.CodePage;
181-
}
182211
if (action.Contains("IsActiveEOL"))
183212
{
184213
Int32 eolMode = document.SciControl.EOLMode;
185214
if (((ItemData)item.Tag).Tag != eolMode.ToString()) return false;
186215
}
187-
if (action.Contains("SyntaxIs?"))
216+
if (action.Contains("IsDefaultEncoding"))
188217
{
189-
String[] chunks = action.Split('?');
190-
if (chunks.Length == 2)
191-
{
192-
String language = document.SciControl.ConfigurationLanguage;
193-
if (chunks[chunks.Length - 1] != language.ToUpper()) return false;
194-
}
218+
Int32 codepage = document.SciControl.Encoding.CodePage;
219+
if (codepage != Encoding.Default.CodePage) return false;
195220
}
196221
}
197-
if (action.Contains("!IsFullScreen"))
198-
{
199-
if (MainForm.Instance.IsFullScreen) return false;
200-
}
201-
else if (action.Contains("IsFullScreen"))
202-
{
203-
if (!MainForm.Instance.IsFullScreen) return false;
204-
}
205-
if (action.Contains("TracksBoolean"))
206-
{
207-
Boolean value = (Boolean)Globals.Settings.GetValue(((ItemData)item.Tag).Tag);
208-
if (!value) return false;
209-
}
210222
return true;
211223
}
212224

@@ -302,25 +314,42 @@ public static String GetActiveEncodingName()
302314
ITabbedDocument document = Globals.CurrentDocument;
303315
if (document != null && document.IsEditable)
304316
{
305-
Boolean hasBOM = document.SciControl.SaveBOM;
306317
Int32 codepage = document.SciControl.Encoding.CodePage;
307-
if (codepage == Encoding.UTF8.CodePage)
308-
{
309-
return GetLabelAsPlainText("Label.UTF8", hasBOM);
310-
}
311-
else if (codepage == Encoding.UTF7.CodePage)
312-
{
313-
return GetLabelAsPlainText("Label.UTF7", hasBOM);
314-
}
315-
else if (codepage == Encoding.BigEndianUnicode.CodePage)
318+
EncodingFileInfo info = FileHelper.GetEncodingFileInfo(document.FileName);
319+
if (codepage == info.CodePage)
316320
{
317-
return GetLabelAsPlainText("Label.BigEndian", hasBOM);
321+
if (ScintillaManager.IsUnicode(info.CodePage))
322+
{
323+
String name = "Unicode (" + info.Charset + ")";
324+
return info.ContainsBOM ? name + " (BOM)" : name;
325+
}
326+
else
327+
{
328+
String name = TextHelper.GetString("Label.8Bits").Replace("&", "");
329+
return name + " (" + info.Charset + ")";
330+
}
318331
}
319-
else if (codepage == Encoding.Unicode.CodePage)
332+
else // Opened in different encoding...
320333
{
321-
return GetLabelAsPlainText("Label.LittleEndian", hasBOM);
334+
Boolean hasBOM = document.SciControl.SaveBOM;
335+
if (codepage == Encoding.UTF8.CodePage)
336+
{
337+
return GetLabelAsPlainText("Label.UTF8", hasBOM);
338+
}
339+
else if (codepage == Encoding.UTF7.CodePage)
340+
{
341+
return GetLabelAsPlainText("Label.UTF7", hasBOM);
342+
}
343+
else if (codepage == Encoding.BigEndianUnicode.CodePage)
344+
{
345+
return GetLabelAsPlainText("Label.BigEndian", hasBOM);
346+
}
347+
else if (codepage == Encoding.Unicode.CodePage)
348+
{
349+
return GetLabelAsPlainText("Label.LittleEndian", hasBOM);
350+
}
351+
else return GetLabelAsPlainText("Label.8Bits", false);
322352
}
323-
else return GetLabelAsPlainText("Label.8Bits", false);
324353
}
325354
else return TextHelper.GetString("Info.Unknown");
326355
}

‎FlashDevelop/Managers/ScintillaManager.cs

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -118,25 +118,15 @@ public static void MakeFileWritable(ScintillaControl sci)
118118
}
119119

120120
/// <summary>
121-
/// Selects a correct codepage for the editor
121+
/// Detects from codepage if the encoding is unicode
122122
/// </summary>
123-
public static Int32 SelectCodePage(Int32 codepage)
123+
public static Boolean IsUnicode(Int32 codepage)
124124
{
125-
if (codepage == 65001 /*UTF-8*/
126-
|| codepage == 1200 /*UTF-16LE*/
127-
|| codepage == 1201) /*UTF-16BE*/
128-
{
129-
return 65001;
130-
}
131-
else if (codepage == 932 /*Japanese Shift JIS*/
132-
|| codepage == 936 /*Simplified Chinese GBK*/
133-
|| codepage == 949 /*Korean Unified Hangul Code*/
134-
|| codepage == 950 /*Traditional Chinese Big5*/
135-
|| codepage == 1361 /*Korean Johab*/)
136-
{
137-
return codepage;
138-
}
139-
else return 0; // Disable multibyte support
125+
return (codepage == Encoding.UTF7.CodePage
126+
|| codepage == Encoding.UTF8.CodePage
127+
|| codepage == Encoding.UTF32.CodePage
128+
|| codepage == Encoding.BigEndianUnicode.CodePage
129+
|| codepage == Encoding.Unicode.CodePage);
140130
}
141131

142132
/// <summary>
@@ -384,9 +374,9 @@ public static ScintillaControl CreateControl(String file, String text, Int32 cod
384374
sci.SetXCaretPolicy((Int32)(CaretPolicy.Jumps | CaretPolicy.Even), 30);
385375
sci.SetYCaretPolicy((Int32)(CaretPolicy.Jumps | CaretPolicy.Even), 2);
386376
sci.ScrollWidthTracking = (Globals.Settings.ScrollWidth == 3000);
387-
sci.CodePage = SelectCodePage(codepage);
377+
sci.CodePage = 65001; // Editor handles text as UTF-8
388378
sci.Encoding = Encoding.GetEncoding(codepage);
389-
sci.SaveBOM = (sci.CodePage == 65001) && Globals.Settings.SaveUnicodeWithBOM;
379+
sci.SaveBOM = IsUnicode(codepage) && Globals.Settings.SaveUnicodeWithBOM;
390380
sci.Text = text; sci.FileName = file; // Set text and save file name
391381
sci.Modified += new ModifiedHandler(Globals.MainForm.OnScintillaControlModified);
392382
sci.MarginClick += new MarginClickHandler(Globals.MainForm.OnScintillaControlMarginClick);

‎PluginCore/PluginCore/Helpers/FileHelper.cs

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,47 +339,72 @@ public static EncodingFileInfo GetEncodingFileInfo(String file)
339339
startIndex = 3;
340340
info.BomLength = 3;
341341
info.ContainsBOM = true;
342+
info.Charset = Encoding.UTF8.WebName;
342343
info.CodePage = Encoding.UTF8.CodePage;
343344
}
344345
else if (bytes.Length > 3 && (bytes[0] == 0xff && bytes[1] == 0xfe && bytes[2] == 0x00 && bytes[3] == 0x00))
345346
{
346347
startIndex = 4;
347348
info.BomLength = 4;
348349
info.ContainsBOM = true;
350+
info.Charset = Encoding.UTF32.WebName;
349351
info.CodePage = Encoding.UTF32.CodePage;
350352
}
351353
else if (bytes.Length > 4 && ((bytes[0] == 0x2b && bytes[1] == 0x2f && bytes[2] == 0x76) && (bytes[3] == 0x38 || bytes[3] == 0x39 || bytes[3] == 0x2b || bytes[3] == 0x2f) && bytes[4] == 0x2D))
352354
{
353355
startIndex = 5;
354356
info.BomLength = 5;
355357
info.ContainsBOM = true;
358+
info.Charset = Encoding.UTF7.WebName;
356359
info.CodePage = Encoding.UTF7.CodePage;
357360
}
358361
else if (bytes.Length > 3 && ((bytes[0] == 0x2b && bytes[1] == 0x2f && bytes[2] == 0x76) && (bytes[3] == 0x38 || bytes[3] == 0x39 || bytes[3] == 0x2b || bytes[3] == 0x2f)))
359362
{
360363
startIndex = 4;
361364
info.BomLength = 4;
362365
info.ContainsBOM = true;
366+
info.Charset = Encoding.UTF7.WebName;
363367
info.CodePage = Encoding.UTF7.CodePage;
364368
}
365369
else if (bytes.Length > 1 && (bytes[0] == 0xff && bytes[1] == 0xfe))
366370
{
367371
startIndex = 2;
368372
info.BomLength = 2;
369373
info.ContainsBOM = true;
374+
info.Charset = Encoding.Unicode.WebName;
370375
info.CodePage = Encoding.Unicode.CodePage;
371376
}
372377
else if (bytes.Length > 1 && (bytes[0] == 0xfe && bytes[1] == 0xff))
373378
{
374379
startIndex = 2;
375380
info.BomLength = 2;
376381
info.ContainsBOM = true;
382+
info.Charset = Encoding.BigEndianUnicode.WebName;
377383
info.CodePage = Encoding.BigEndianUnicode.CodePage;
378384
}
379385
else
380386
{
381-
if (!ContainsInvalidUTF8Bytes(bytes)) info.CodePage = Encoding.UTF8.CodePage;
382-
else info.CodePage = Encoding.Default.CodePage;
387+
if (!ContainsInvalidUTF8Bytes(bytes))
388+
{
389+
info.Charset = Encoding.UTF8.WebName;
390+
info.CodePage = Encoding.UTF8.CodePage;
391+
}
392+
else // Try detecting using Ude...
393+
{
394+
Ude.CharsetDetector detector = new Ude.CharsetDetector();
395+
detector.Feed(bytes, 0, bytes.Length); detector.DataEnd();
396+
if (detector.Charset != null)
397+
{
398+
Encoding encoding = Encoding.GetEncoding(detector.Charset);
399+
info.Charset = encoding.WebName;
400+
info.CodePage = encoding.CodePage;
401+
}
402+
else
403+
{
404+
info.Charset = Encoding.Default.WebName;
405+
info.CodePage = Encoding.Default.CodePage;
406+
}
407+
}
383408
}
384409
Int32 contentLength = bytes.Length - startIndex;
385410
if (bytes.Length > 0 && bytes.Length > startIndex)
@@ -439,6 +464,7 @@ public override byte[] GetPreamble()
439464
public class EncodingFileInfo
440465
{
441466
public Int32 CodePage = -1;
467+
public String Charset = String.Empty;
442468
public String Contents = String.Empty;
443469
public Boolean ContainsBOM = false;
444470
public Int32 BomLength = 0;

‎PluginCore/UdeLibrary/Ude.Core/Big5Prober.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,8 @@ public override void Reset()
9696

9797
public override string GetCharsetName()
9898
{
99-
return "Big-5";
99+
//return "Big-5";
100+
return "Big5"; // Correct for Windows
100101
}
101102

102103
public override float GetConfidence()

0 commit comments

Comments
 (0)