Merge pull request #42 from augmentable-dev/add-file-preview

preview file content when identifying language

Author: patrickdevivo

pkg/comments/comments.go

@@ -1,7 +1,9 @@
 package comments
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -24,9 +26,19 @@ type Comment struct {
 
 // SearchFile searches a file for comments. It infers the language
 func SearchFile(filePath string, reader io.Reader, cb func(*Comment)) error {
-	// TODO right now, enry only infers the language based on the file extension
-	// we should add some "preview" bytes from the file so that it has some sample content to examine
-	lang := Language(enry.GetLanguage(filepath.Base(filePath), nil))
+	// create a preview reader that reads in some of the file for enry to better identify the language
+	var buf bytes.Buffer
+	tee := io.TeeReader(reader, &buf)
+	previewReader := io.LimitReader(tee, 1000)
+	preview, err := ioutil.ReadAll(previewReader)
+	if err != nil {
+		return err
+	}
+
+	// create a new reader concatenating the preview and the original reader (which has now been read from)
+	fullReader := io.MultiReader(strings.NewReader(buf.String()), reader)
+
+	lang := Language(enry.GetLanguage(filepath.Base(filePath), preview))
 	if enry.IsVendor(filePath) {
 		return nil
 	}
@@ -39,7 +51,7 @@ func SearchFile(filePath string, reader io.Reader, cb func(*Comment)) error {
 		return err
 	}
 
-	collections, err := commentParser.Parse(reader)
+	collections, err := commentParser.Parse(fullReader)
 	if err != nil {
 		return err
 	}

pkg/comments/comments_test.go

@@ -47,3 +47,17 @@ func TestRustFiles(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestPHPFiles(t *testing.T) {
+	var comments Comments
+	err := SearchDir("testdata/php", func(comment *Comment) {
+		comments = append(comments, comment)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(comments) != 3 {
+		t.Fail()
+	}
+}

pkg/comments/languages.go

@@ -49,7 +49,7 @@ var LanguageParseOptions map[Language]*lege.ParseOptions = map[Language]*lege.Pa
 	"JavaScript":   CStyleCommentOptions,
 	"Python":       HashStyleCommentOptions,
 	"Ruby":         HashStyleCommentOptions,
-	"PHP":          CStyleCommentOptions,
+	"PHP":          {Boundaries: append(CStyleCommentOptions.Boundaries, HashStyleCommentOptions.Boundaries...)},
 	"Shell":        HashStyleCommentOptions,
 	"Visual Basic": {Boundaries: []lege.Boundary{{Start: "'", End: "\n"}}},
 	"TypeScript":   CStyleCommentOptions,

pkg/comments/testdata/php/test.php

@@ -0,0 +1,8 @@
+<?php
+// this is a comment
+
+/* this is another comment */
+
+# and one last one
+?>
+