HayooCrawler with simple JSON support added

hayooCrawler/.gitignore

@@ -0,0 +1 @@
+dist

hayooCrawler/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2007 Sebastian M. Schlatt, Timo B. Hübel
+
+Permission is hereby granted, free of charge, to any person obtaining 
+a copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 
+sell copies of the Software, and to permit persons to whom the Software is 
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

hayooCrawler/Makefile

@@ -0,0 +1,8 @@
+all	:
+	 cabal configure && cabal build && cabal install
+
+pkg	:
+	 $(MAKE) -C data pkg
+
+.PHONY	: all pkg
+

hayooCrawler/data/.gitignore

@@ -0,0 +1,4 @@
+cache
+packages
+tmp
+pkg.tgz

hayooCrawler/data/Makefile

@@ -0,0 +1,36 @@
+
+# crawl all packages and store .json data per package in subdir packages
+# 0000-ranks.json contains the json data for updating the packages
+# with the package rank. Packages with default value (1.0) for rank
+# are not updated
+
+pkg	:
+	[ -d tmp ]      || mkdir tmp
+	[ -d packages ] || mkdir packages
+	hayooCrawler --pkg-index -j --ranking
+
+# save and compress the package files
+
+tar	: pkg.tgz
+
+# restore saved package files
+
+untar	:
+	tar xvzf pkg.tgz
+
+# load and parse all hackage pages
+# and store them in binary format in
+# dir cache. Usefull for test runs
+# reduces web trafic.
+
+cache	:
+	hayooCrawler --cache
+
+clean	:
+	rm -rf tmp packages
+
+.PHONY	: pkg tar untar cache clean
+
+pkg.tgz	:
+	tar cvzf pkg.tgz packages
+

hayooCrawler/hayooCrawler.cabal

@@ -0,0 +1,125 @@
+name:          hayooCrawler
+version:       0.0.0.0
+license:       MIT
+license-file:  LICENSE
+author:        Uwe Schmidt
+copyright:     Copyright (c) 2007 - 2013 Uwe Schmidt, Sebastian M. Gauck and Timo B. Kranz
+maintainer:    Uwe Schmidt <[email protected]>
+stability:     experimental
+category:      Text, Data
+synopsis:      The Hayoo! crawler for indexing hackage documentation
+description:   The Hayoo! crawler for indexing hackage documentation
+               .
+               Initial version: Derived from holumbus repo
+cabal-version: >=1.10
+build-type:    Simple
+-- tested-with:  ghc-7.6
+
+-- ------------------------------------------------------------
+
+executable hayooCrawler
+  main-is:         HayooCrawler.hs
+
+  other-modules:   Control.Monad.ReaderStateIO
+                 , Control.Sequential.MapFoldBinary
+                 , Hayoo.FctIndexerCore
+                 , Hayoo.FunctionInfo
+                 , Hayoo.HackagePackage
+                 , Hayoo.Haddock   
+                 , Hayoo.IndexConfig
+                 , Hayoo.IndexTypes
+                 , Hayoo.PackageArchive
+                 , Hayoo.PackageInfo
+                 , Hayoo.PkgIndexerCore
+                 , Hayoo.PackageRank
+                 , Hayoo.Signature 
+                 , Hayoo.URIConfig 
+                 , Holumbus.Crawler
+                 , Holumbus.Crawler.CacheCore
+                 , Holumbus.Crawler.Constants
+                 , Holumbus.Crawler.Core
+                 , Holumbus.Crawler.Html
+                 , Holumbus.Crawler.IndexerCore
+                 , Holumbus.Crawler.Logger
+                 , Holumbus.Crawler.Robots
+                 , Holumbus.Crawler.RobotTypes
+                 , Holumbus.Crawler.Types
+                 , Holumbus.Crawler.URIs
+                 , Holumbus.Crawler.Util
+                 , Holumbus.Crawler.XmlArrows
+                 , Holumbus.Data.Crunch
+                 , Holumbus.Data.PrefixTree
+                 , Holumbus.Data.PrefixTree.Core
+                 , Holumbus.Data.PrefixTree.FuzzySearch
+                 , Holumbus.Data.PrefixTree.PrefixSet
+                 , Holumbus.Data.PrefixTree.Types
+                 , Holumbus.Index.Common
+                 , Holumbus.Index.Common.BasicTypes
+                 , Holumbus.Index.Common.DiffList
+                 , Holumbus.Index.Common.DocId
+                 , Holumbus.Index.Common.DocIdMap
+                 , Holumbus.Index.Common.Document
+                 , Holumbus.Index.Common.LoadStore
+                 , Holumbus.Index.Common.Occurences
+                 , Holumbus.Index.Common.RawResult
+                 , Holumbus.Index.CompactDocuments
+                 , Holumbus.Index.CompactIndex
+                 , Holumbus.Index.CompactSmallDocuments
+                 , Holumbus.Index.Compression
+                 , Holumbus.Index.Inverted.CompressedPrefixMem
+                 , Holumbus.Query.Result
+                 , Holumbus.Utility
+
+  build-depends:   base                   >= 4     && < 5
+                 , aeson                  >= 0.6
+                 , aeson-pretty           >= 0.7
+                 , binary                 >= 0.5
+                 , bytestring             >= 0.9
+                 , bzlib                  >= 0.4   && < 1
+                 , containers             >= 0.4.2
+                 , deepseq                >= 1.2
+                 , directory              >= 1.1
+                 , enummapset             >= 0     && < 1
+                 , filepath               >= 1     && < 2
+                 , hslogger               >= 1     && < 2
+                 -- , Holumbus-Searchengine  >= 2     && < 3
+                 , hxt                    >= 9.1   && < 10
+                 , hxt-cache              >= 9     && < 10
+                 , hxt-charproperties     >= 9     && < 10
+                 , hxt-curl               >= 9     && < 10
+                 , hxt-http               >= 9     && < 10
+                 , hxt-regex-xmlschema    >= 9     && < 10
+                 , hxt-unicode            >= 9     && < 10
+                 , hxt-xpath              >= 9.1   && < 10
+                 , mtl                    >= 1.1   && < 3
+                 , network                >= 2.1   && < 3
+                 , old-time               >= 1     && < 2
+                 , parsec                 >= 2.1   && < 4
+                 , parallel               >= 3.1   && < 4
+                 , process                >= 1     && < 2
+                 , tar                    >= 0.3   && < 1
+                 , text                   >= 0.11
+                 , zlib                   >= 0.5   && < 1 
+                 , SHA                    >= 1.4   && < 2
+                 , unix                   >= 2.3   && < 3
+                 , unordered-containers   >= 0.2
+
+  hs-source-dirs: src
+
+  ghc-options:   -threaded
+                 -Wall
+                 -funbox-strict-fields
+                 -rtsopts
+                 -fwarn-tabs
+
+  default-language: Haskell2010
+
+-- ------------------------------------------------------------
+
+source-repository head
+  type:     git
+  location: https://github.com/UlfS/holumbus.git
+
+-- ------------------------------------------------------------
+
+

hayooCrawler/src/.gitignore

@@ -0,0 +1,4 @@
+cache
+functions
+packages
+tmp

hayooCrawler/src/Control/Monad/ReaderStateIO.hs

@@ -0,0 +1,61 @@
+{-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE FlexibleInstances #-}
+
+-- ------------------------------------------------------------
+
+module Control.Monad.ReaderStateIO
+    ( module Control.Monad.ReaderStateIO
+    )
+where
+
+import          Control.Monad.Reader
+import          Control.Monad.State
+
+-- ------------------------------------------------------------
+
+-- |
+-- reader state io monad implemented directly without any monad transformers
+
+newtype ReaderStateIO env state res = RSIO ( env -> state -> IO (res, state) )
+
+instance Monad (ReaderStateIO env state) where
+    return v
+        = RSIO $ \ _e s -> return (v, s)
+
+    RSIO cmd >>= f
+        = RSIO $ \ e s -> do
+                          (r', s') <- cmd e s
+                          let RSIO cmd2 = f r'
+                          s' `seq` cmd2 e s'
+
+instance MonadIO (ReaderStateIO env state) where
+    liftIO a
+        = RSIO $ \ _e s -> do
+                           r <- a
+                           return (r, s)
+
+instance MonadState state (ReaderStateIO env state) where
+    get
+        = RSIO $ \ _e  s -> return (s, s)
+
+    put s
+        = RSIO $ \ _e _s -> return ((), s)
+
+instance MonadReader env (ReaderStateIO env state) where
+    ask
+        = RSIO $ \  e  s -> return (e, s)
+
+    local f (RSIO cmd)
+        = RSIO $ \  e  s -> cmd (f e) s
+
+modifyIO                :: (state -> IO state) -> ReaderStateIO env state ()
+modifyIO f              = do
+                          s0 <- get
+                          s1 <- liftIO (f s0)
+                          put s1
+
+runReaderStateIO        :: ReaderStateIO env state res  -> env -> state -> IO (res, state)
+runReaderStateIO (RSIO cmd) e s
+    = cmd e s
+
+-- ------------------------------------------------------------

hayooCrawler/src/Control/Sequential/MapFoldBinary.hs

@@ -0,0 +1,127 @@
+{-# OPTIONS #-}
+
+-- ------------------------------------------------------------
+
+{- |
+   Module     : Control.Sequential.MapFoldBinary
+   Copyright  : Copyright (C) 2010 Uwe Schmidt
+   License    : MIT
+
+   Maintainer : Uwe Schmidt ([email protected])
+   Stability  : experimental
+
+   A map-fold function for interleaved map and fold.
+   The elements of a list are processed like in a binary tree.
+
+-}
+
+-- ------------------------------------------------------------
+
+module Control.Sequential.MapFoldBinary
+    ( mapFoldBinary
+    , mapFoldBinaryM
+    )
+where
+
+-- ------------------------------------------------------------
+
+-- | Pure version of binary map fold
+--
+-- @mapFoldBinary id (+) [1..8]@ adds the elements of a list in the following order:
+-- @(((1+2)+(3+4))+((5+6)+(7+8)))@
+
+mapFoldBinary                   :: (a -> b) -> (b -> b -> b) -> [a] -> b
+mapFoldBinary m f xs0           = fst $ mapLeft (1::Int) $ map1 xs0
+    where
+    -- map1                     :: [a] -> (b, [a])
+    map1 (x:xs)                 = (m x, xs)
+    map1 []                     = error "mapFoldBinary with empty list"
+
+    -- mapLeft                  :: Int -> (b, [a]) -> (b, [a])
+    mapLeft n (r1, xs1)
+        | null xs1              = (r1, [])
+        | null xs2              = (r,  [])
+        | otherwise             = mapLeft (n + 1) (r, xs2)
+        where
+        (r2, xs2)               = mapRight n xs1
+        r                       = f r1 r2
+
+    -- mapRight                 :: Int -> [a] -> (b, [a])
+    mapRight 1 xs               = map1 xs
+    mapRight n xs
+        | null xs1              = (r1,      [] )
+        | otherwise             = (f r1 r2, xs2)
+        where
+        (r1, xs1)               = mapRight (n - 1) xs
+        (r2, xs2)               = mapRight (n - 1) xs1
+
+-- ------------------------------------------------------------
+{-
+t1      :: [Int] -> String
+t1      = mapFoldBinary show (\ x y -> "(" ++ x ++ "+" ++ y ++ ")")
+
+r1      :: String
+r1      = t1 [1..8]
+-}
+-- ------------------------------------------------------------
+
+-- | Monadic version of a binary map fold
+--
+-- The elements of a list are mapped and folded in the same way as in the pure version.
+-- The map and fold operations are interleaved. In the above example the expressions are evaluated
+-- from left to right, folding is performed, as early as possible.
+
+
+mapFoldBinaryM                  :: (Monad m) =>
+                                   (a -> m b) -> (b -> b -> m b) -> [a] -> m b
+mapFoldBinaryM m f xs0          = do
+                                  r0 <- map1 xs0
+                                  rn <- mapLeft (1::Int) r0
+                                  return (fst rn)
+    where
+    -- map1                     :: [a] -> m (b, [a])
+    map1 (x:xs)                 = do
+                                  r1 <- m x
+                                  return (r1, xs)
+    map1 []                     = error "mapFoldBinary with empty list"
+
+    -- mapLeft                  :: Int -> (b, [a]) -> m (b, [a])
+    mapLeft n r@(r1, xs1)
+        | null xs1              = return r
+        | otherwise             = do
+                                  (r2, xs2) <- mapRight n xs1
+                                  res       <- f r1 r2
+                                  ( if null xs2
+                                    then return
+                                    else mapLeft (n + 1) ) (res, xs2)
+
+    -- mapRight                 :: Int -> [a] -> m (b, [a])
+    mapRight 1 xs               = map1 xs
+    mapRight n xs               = do
+                                  r@(r1, xs1) <- mapRight (n - 1) xs
+                                  if null xs1
+                                     then return r
+                                     else do
+                                          (r2, xs2) <- mapRight (n - 1) xs1
+                                          res       <- f r1 r2
+                                          return (res, xs2)
+
+-- ------------------------------------------------------------
+{-
+t1m     :: [Int] -> IO String
+t1m     = mapFoldBinaryM
+          ( \ x -> ( do
+                     print x
+                     return (show x)
+                   )
+          )
+          ( \ x y -> ( do
+                       putStrLn $ "(" ++ x ++ "+" ++ y ++ ")"
+                       return   $ "(" ++ x ++ "+" ++ y ++ ")"
+                     )
+          )
+
+r1m     :: IO String
+r1m     = t1m [1..8]
+-}
+-- ------------------------------------------------------------