Parsing example

Author: newhoggy

README.md

@@ -5,27 +5,8 @@
 succinct data-structures to allow traversal of large XML
 strings with minimal memory overhead.
 
-For an example, see app/Main.hs
+For an example, see [app/Main.hs](../master/app/Main.hs)
 
-```
-benchmarking XmlBig/Run blankXml
-time                 2.212 s    (1.971 s .. 2.652 s)
-                     0.996 R²   (0.989 R² .. 1.000 R²)
-mean                 2.138 s    (2.073 s .. 2.186 s)
-std dev              73.37 ms   (0.0 s .. 83.51 ms)
-variance introduced by outliers: 19% (moderately inflated)
-
-benchmarking XmlBig/Run xmlToInterestBits3
-time                 2.497 s    (2.449 s .. 2.531 s)
-                     1.000 R²   (1.000 R² .. 1.000 R²)
-mean                 2.531 s    (2.515 s .. 2.540 s)
-std dev              13.90 ms   (0.0 s .. 14.76 ms)
-variance introduced by outliers: 19% (moderately inflated)
-
-benchmarking XmlBig/loadXml
-time                 2.768 s    (2.698 s .. 2.857 s)
-                     1.000 R²   (1.000 R² .. 1.000 R²)
-mean                 2.780 s    (2.767 s .. 2.790 s)
-std dev              15.40 ms   (0.0 s .. 17.48 ms)
-variance introduced by outliers: 19% (moderately inflated)
-```
+# Notes
+* [Semi-Indexing Semi-Structured Data in Tiny Space](http://www.di.unipi.it/~ottavian/files/semi_index_cikm.pdf)
+* [Space-Efficient, High-Performance Rank & Select Structures on Uncompressed Bit Sequences](https://www.cs.cmu.edu/~dga/papers/zhou-sea2013.pdf)

app/Main.hs

@@ -1,8 +1,12 @@
-{-# LANGUAGE BangPatterns        #-}
-{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE BangPatterns         #-}
+{-# LANGUAGE FlexibleInstances    #-}
+{-# LANGUAGE ScopedTypeVariables  #-}
+{-# LANGUAGE TypeSynonymInstances #-}
 
 module Main where
 
+import Data.Foldable
+import Data.Maybe
 import Data.Semigroup                                ((<>))
 import Data.Word
 import HaskellWorks.Data.BalancedParens.RangeMinMax2
@@ -11,37 +15,98 @@ import HaskellWorks.Data.Bits.BitShown
 import HaskellWorks.Data.FromByteString
 import HaskellWorks.Data.RankSelect.CsPoppy
 import HaskellWorks.Data.TreeCursor
+import HaskellWorks.Data.Xml.Decode
+import HaskellWorks.Data.Xml.DecodeResult
+import HaskellWorks.Data.Xml.RawDecode
 import HaskellWorks.Data.Xml.RawValue
 import HaskellWorks.Data.Xml.Succinct.Cursor
 import HaskellWorks.Data.Xml.Succinct.Index
+import HaskellWorks.Data.Xml.Value
 
 import qualified Data.ByteString      as BS
 import qualified Data.Vector.Storable as DVS
 
 type RawCursor = XmlCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64))
-type CsCursor = XmlCursor BS.ByteString CsPoppy (RangeMinMax2 CsPoppy)
+type FastCursor = XmlCursor BS.ByteString CsPoppy (RangeMinMax2 CsPoppy)
 
+-- | Read an XML file into memory and return a raw cursor initialised to the
+-- start of the XML document.
 readRawCursor :: String -> IO RawCursor
 readRawCursor path = do
   !bs <- BS.readFile path
   let !cursor = fromByteString bs :: RawCursor
   return cursor
 
-readCsCursor :: String -> IO CsCursor
-readCsCursor filename = do
+-- | Read an XML file into memory and return a query-optimised cursor initialised
+-- to the start of the XML document.
+readFastCursor :: String -> IO FastCursor
+readFastCursor filename = do
+  -- Load the XML file into memory as a raw cursor.
+  -- The raw XML data is `text`, and `ib` and `bp` are the indexes.
+  -- `ib` and `bp` can be persisted to an index file for later use to avoid
+  -- re-parsing the file.
   XmlCursor !text (BitShown !ib) (SimpleBalancedParens !bp) _ <- readRawCursor filename
   let !bpCsPoppy = makeCsPoppy bp
   let !rangeMinMax = mkRangeMinMax2 bpCsPoppy
   let !ibCsPoppy = makeCsPoppy ib
   return $ XmlCursor text ibCsPoppy rangeMinMax 1
 
+-- | Parse the text of an XML node.
+class ParseText a where
+  parseText :: Value -> DecodeResult a
+
+instance ParseText String where
+  parseText (XmlText text)      = DecodeOk text
+  parseText (XmlCData text)     = DecodeOk text
+  parseText (XmlElement _ _ cs) = DecodeOk $ concat $ concat $ toList . parseText <$> cs
+  parseText _                   = DecodeOk ""
+
+-- | Convert a decode result to a maybe
+decodeResultToMaybe :: DecodeResult a -> Maybe a
+decodeResultToMaybe (DecodeOk a) = Just a
+decodeResultToMaybe _            = Nothing
+
+-- | Document model.  This does not need to be able to completely represent all
+-- the data in the XML document.  In fact, having a smaller model may improve
+-- query performance.
+data Plant = Plant
+  { common :: String
+  , price  :: String
+  } deriving (Eq, Show)
+
+newtype Catalog = Catalog
+  { plants :: [Plant]
+  } deriving (Eq, Show)
+
+-- | Decode plant element
+decodePlant :: Value -> DecodeResult Plant
+decodePlant xml = do
+  aCommon <- xml /> "common"  >>= parseText
+  aPrice  <- xml /> "price"   >>= parseText
+  return $ Plant aCommon aPrice
+
+-- | Decode catalog element
+decodeCatalog :: Value -> DecodeResult Catalog
+decodeCatalog xml = do
+  aPlantXmls <- xml />> "plant"
+  let aPlants = catMaybes (decodeResultToMaybe . decodePlant <$> aPlantXmls)
+  return $ Catalog aPlants
+
 main :: IO ()
 main = do
-  !cursor <- readCsCursor "data/catalog.xml"
+  -- Read XML into memory as a query-optimised cursor
+  !cursor <- readFastCursor "data/catalog.xml"
+  -- Skip the XML declaration to get to the root element cursor
   case nextSibling cursor of
     Just rootCursor -> do
+      -- Get the root raw XML value at the root element cursor
       let rootValue = rawValueAt (xmlIndexAt rootCursor)
+      -- Show what we have at this cursor
       putStrLn $ "Raw value: " <> take 100 (show rootValue)
+      -- Decode the raw XML value
+      case decodeCatalog (rawDecode rootValue) of
+        DecodeOk catalog -> putStrLn $ "Catalog: " <> show catalog
+        DecodeFailed msg -> putStrLn $ "Error: " <> show msg
     Nothing -> do
       putStrLn "Could not read XML"
       return ()

stack.yaml

@@ -4,6 +4,16 @@ packages:
 - '.'
 
 extra-deps:
+  - hw-balancedparens-0.2.0.1
+  - hw-bits-0.7.0.1
+  - hw-conduit-0.2.0.2
+  - hw-excess-0.2.0.0
+  - hw-int-0.0.0.3
+  - hw-parser-0.0.0.3
+  - hw-prim-0.5.0.0
+  - hw-rankselect-0.10.0.3
+  - hw-rankselect-base-0.2.0.2
+  - hw-string-parse-0.0.0.4
   - hedgehog-0.5
 
 flags: {}