Last fixes before release (#94)

* Remove allBlockRanges

* When downloading files, create a separate directory for each Unicode version. This facilitate the work on various Unicode versions.

* Improve docs

Author: wismill

README.md

@@ -66,7 +66,7 @@ See `unicode-data`’s [guide](unicode-data/README.md#unicode-database-version-u
 ## Licensing
 
 `unicode-data*` packages are an [open source](https://github.com/composewell/unicode-data)
-project available under a liberal [Apache-2.0 license](LICENSE).
+project available under a liberal [Apache-2.0 license](unicode-data/LICENSE).
 
 ## Contributing
 

experimental/unicode-data-text/lib/Unicode/Text/Case.hs

@@ -55,14 +55,17 @@ streamUnfold (C.Unfold step inject) = \case
 caseConvertStream :: C.Unfold Char Char -> T.Text -> T.Text
 caseConvertStream u t = TF.unstream (streamUnfold u (TF.stream t))
 
+-- | Convert to full upper case using 'T.Text' fusion.
 {-# INLINE toUpperFusion #-}
 toUpperFusion :: T.Text -> T.Text
 toUpperFusion = caseConvertStream C.upperCaseMapping
 
+-- | Convert to full lower case using 'T.Text' fusion.
 {-# INLINE toLowerFusion #-}
 toLowerFusion :: T.Text -> T.Text
 toLowerFusion = caseConvertStream C.lowerCaseMapping
 
+-- | Convert to full case fold using 'T.Text' fusion.
 {-# INLINE toCaseFoldFusion #-}
 toCaseFoldFusion :: T.Text -> T.Text
 toCaseFoldFusion = caseConvertStream C.caseFoldMapping
@@ -105,6 +108,7 @@ streamUnfoldToTitle = case C.lowerCaseMapping of
                   C.Yield c st' -> TF.Yield c (CC2 s st')
 {-# INLINE [0] streamUnfoldToTitle #-}
 
+-- | Convert to full title case using 'T.Text' fusion.
 {-# INLINE toTitleFusion #-}
 toTitleFusion :: T.Text -> T.Text
 toTitleFusion = TF.unstream . streamUnfoldToTitle . TF.stream
@@ -189,18 +193,21 @@ caseConvertText ascii (C.Unfold (step :: u -> C.Step u Char) inject) (T.Text src
                 writeMapping (step st) (dstOff + d)
 {-# INLINE caseConvertText #-}
 
+-- | Convert to full upper case /without/ 'T.Text' fusion.
 {-# INLINE toUpper #-}
 toUpper :: T.Text -> T.Text
 toUpper = caseConvertText
     (\w -> if w - 97 <= 25 then w - 32 else w)
     C.upperCaseMapping
 
+-- | Convert to full lower case /without/ 'T.Text' fusion.
 {-# INLINE toLower #-}
 toLower :: T.Text -> T.Text
 toLower = caseConvertText
     (\w -> if w - 65 <= 25 then w + 32 else w)
     C.lowerCaseMapping
 
+-- | Convert to full case fold /without/ 'T.Text' fusion.
 {-# INLINE toCaseFold #-}
 toCaseFold :: T.Text -> T.Text
 toCaseFold = caseConvertText

experimental/unicode-data-text/unicode-data-text.cabal

@@ -4,8 +4,8 @@ version:             0.1.0
 synopsis:            Unicode features for “text” package
 description:
   @unicode-data-text@ provides Unicode features from
-  <https://hackage.haskell.org/package/unicode-data @unicode-data@> package
-  for the <https://hackage.haskell.org/package/text @text@> package.
+  <https://hackage.haskell.org/package/unicode-data unicode-data> package
+  for the <https://hackage.haskell.org/package/text text> package.
 homepage:            http://github.com/composewell/unicode-data
 bug-reports:         https://github.com/composewell/unicode-data/issues
 license:             Apache-2.0

ucd.sh

@@ -44,7 +44,7 @@ SECURITY_FILES="\
 # and verify the $checksum if $VERIFY_CHECKSUM is enabled
 # $1 = file:checksum
 download_file() {
-    local directory="data/$1"
+    local directory="data/$VERSION/$1"
     local url="$2"
     local pair="$3"
     local file="$(echo "$pair" | cut -f1 -d':')"
@@ -81,7 +81,7 @@ download_files() {
 run_generator() {
     # Compile and run ucd2haskell
     cabal run --flag ucd2haskell ucd2haskell -- \
-          --input ./data/ \
+          --input "./data/$VERSION" \
           --output-core ./unicode-data/lib/ \
           --output-names ./unicode-data-names/lib/ \
           --output-scripts ./unicode-data-scripts/lib/ \

unicode-data/bench/Main.hs

@@ -134,9 +134,6 @@ main = defaultMain
     , bgroup "blockDefinition"
       [ benchRangeNF "unicode-data"  (show . B.blockDefinition)
       ]
-    , bgroup "allBlockRanges"
-      [ benchChars "unicode-data"  (const B.allBlockRanges)
-      ]
     ]
   , bgroup "Unicode.Char.General.Compat"
     [ bgroup' "isAlpha"

unicode-data/exe/Parser/Text.hs

@@ -315,7 +315,7 @@ genBlocksModule moduleName = done <$> Fold.foldl' step initial
         , "{-# OPTIONS_HADDOCK hide #-}"
         , ""
         , "module " <> moduleName
-        , "(Block(..), BlockDefinition(..), block, blockDefinition, allBlockRanges)"
+        , "(Block(..), BlockDefinition(..), block, blockDefinition)"
         , "where"
         , ""
         , "import Data.Ix (Ix)"
@@ -344,14 +344,6 @@ genBlocksModule moduleName = done <$> Fold.foldl' step initial
         , "blockDefinition :: Block -> BlockDefinition"
         , "blockDefinition b = case b of"
         , mconcat (reverse defs)
-        , "-- | All the block ranges, in ascending order."
-        , "--"
-        , "-- @since 0.3.1"
-        , "{-# INLINE allBlockRanges #-}"
-        , "allBlockRanges :: [(Int, Int)]"
-        , "allBlockRanges ="
-        , "    " <> show ranges'
-        , ""
         , "-- | Character block, if defined."
         , "--"
         , "-- @since 0.3.1"

unicode-data/lib/Unicode/Char.hs

@@ -35,7 +35,7 @@ module Unicode.Char
     , module Unicode.Char.Identifiers
     , unicodeVersion
 
-    -- * Re-export
+    -- * Re-export from @base@
     , ord
     , chr
     )

unicode-data/lib/Unicode/Char/Case.hs

@@ -9,6 +9,12 @@
 --
 -- Case and case mapping related functions.
 --
+-- This module provides /full/ predicates and mappings that are /not/ compatible
+-- with those in "Data.Char", which rely on simple properties.
+-- See "Unicode.Char.Case.Compat" for a drop-in replacement of the functions in
+-- "Data.Char".
+--
+
 module Unicode.Char.Case
     ( -- * Predicates
       isLowerCase
@@ -48,29 +54,49 @@ import qualified Unicode.Internal.Char.SpecialCasing.LowerCaseMapping as C
 import qualified Unicode.Internal.Char.SpecialCasing.TitleCaseMapping as C
 import qualified Unicode.Internal.Char.SpecialCasing.UpperCaseMapping as C
 
--- | Returns 'True' for lower-case letters.
+-- | Returns 'True' for lower-case characters.
+--
+-- It uses the character property
+-- <https://www.unicode.org/reports/tr44/#Lowercase Lowercase>.
 --
 -- @since 0.3.0
 {-# INLINE isLowerCase #-}
 isLowerCase :: Char -> Bool
 isLowerCase = P.isLowercase
 
--- | Returns 'True' for lower-case letters.
+-- | Returns 'True' for lower-case characters.
+--
+-- It uses the character property
+-- <https://www.unicode.org/reports/tr44/#Lowercase Lowercase>.
 --
 -- @since 0.1.0
 {-# INLINE isLower #-}
 {-# DEPRECATED isLower "Use isLowerCase instead. Note that the behavior of this function does not match base:Data.Char.isLower. See Unicode.Char.Case.Compat for behavior compatible with base:Data.Char." #-}
 isLower :: Char -> Bool
 isLower = P.isLowercase
 
--- | Returns 'True' for upper-case letters.
+-- | Returns 'True' for upper-case characters.
+--
+-- It uses the character property
+-- <https://www.unicode.org/reports/tr44/#Uppercase Uppercase>.
+--
+-- Note: it does /not/ match title-cased letters. Those are matched using:
+-- @'Unicode.Char.General.generalCategory' c ==
+-- 'Unicode.Char.General.TitlecaseLetter'@.
 --
 -- @since 0.3.0
 {-# INLINE isUpperCase #-}
 isUpperCase :: Char -> Bool
 isUpperCase = P.isUppercase
 
--- | Returns 'True' for upper-case letters.
+-- | Returns 'True' for upper-case characters.
+--
+-- It uses the character property
+-- <https://www.unicode.org/reports/tr44/#Uppercase Uppercase>.
+--
+-- Note: it does /not/ match title-cased letters. Those are matched using:
+-- @'Unicode.Char.General.generalCategory' c ==
+-- 'Unicode.Char.General.TitlecaseLetter'@.
 --
 -- @since 0.1.0
 {-# INLINE isUpper #-}

unicode-data/lib/Unicode/Char/Case/Compat.hs

@@ -5,7 +5,7 @@
 -- Maintainer  : [email protected]
 -- Stability   : experimental
 --
--- Compatibility module for case and case mapping related functions..
+-- Compatibility module for case and case mapping related functions.
 --
 -- The functions of this module are drop-in replacement for those in "Data.Char".
 -- They are similar but not identical to some functions in "Unicode.Char.Case",
@@ -30,6 +30,11 @@ import qualified Unicode.Internal.Char.UnicodeData.SimpleUpperCaseMapping as C
 -- Title case is used by a small number of letter ligatures like the
 -- single-character form of /Lj/.
 --
+-- It matches characters with general category 'UppercaseLetter' and
+-- 'TitlecaseLetter'.
+--
+-- See: 'Unicode.Char.Case.isUpperCase' for the /full upper/ case predicate.
+--
 -- prop> isUpper c == Data.Char.isUpper c
 --
 -- @since 0.3.0
@@ -41,6 +46,10 @@ isUpper c = case generalCategory c of
 
 -- | Selects lower-case alphabetic Unicode characters (letters).
 --
+-- It matches characters with general category 'LowercaseLetter'.
+--
+-- See: 'Unicode.Char.Case.isLowerCase' for the /full/ lower case predicate.
+--
 -- prop> isLower c == Data.Char.isLower c
 --
 -- @since 0.3.0

unicode-data/lib/Unicode/Char/General/Blocks.hs

@@ -1,5 +1,5 @@
 -- |
--- Module      : Unicode.Char.General
+-- Module      : Unicode.Char.General.Blocks
 -- Copyright   : (c) 2020 Composewell Technologies and Contributors
 -- License     : Apache-2.0
 -- Maintainer  : [email protected]
@@ -14,7 +14,6 @@ module Unicode.Char.General.Blocks
     , B.BlockDefinition(..)
     , block
     , B.blockDefinition
-    , allBlockRanges
     )
 
 where
@@ -27,11 +26,3 @@ import qualified Unicode.Internal.Char.Blocks as B
 {-# INLINE block #-}
 block :: Char -> Maybe B.Block
 block = fmap toEnum . B.block
-
--- | All the [block](https://www.unicode.org/glossary/#block) ranges,
--- in ascending order.
---
--- @since 0.3.1
-{-# INLINE allBlockRanges #-}
-allBlockRanges :: [(Int, Int)]
-allBlockRanges = B.allBlockRanges