Squashed 'libreadtags/' changes from 203af8750..3f5fba867

3f5fba867 Merge pull request #42 from masatake/revise-test-case
a05721ba2 tests: report failure if TagErrnoFileMaybeTooBig is set
6c2c21a52 Merge pull request #40 from masatake/masatake--lambertlulala-fix/limitation_of_ftell_and_fseek
a222fcf22 pass 0 instead of 0L to fseek() as the beginning of a file stream
4c2f6b2bc update NEWS.md
195173214 Add _WIN32 preprocessor to use _ftelli64 and _fseeki64
167834bd2 Merge pull request #39 from masatake/add-error-const-for-too-large-tags
096152e47 style: delete a redundant usage of 'else' keyword
6dca8a888 Introduce a new error constant TagErrnoFileMaybeTooBig
31e93940d GitHub Actions: show the test log in failure
0f6a0c281 cosmetic: clean up whitespaces
4a6bca8f6 Merge pull request #38 from universal-ctags/masatake-patch-1
e2a4a43c2 GitHub Actions: run test cases on msys2 and mingw
32f4a95eb Merge pull request #35 from masatake/fix-comment
c5182322f comments: fix a typo

git-subtree-dir: libreadtags
git-subtree-split: 3f5fba867cc2c6a5654ab8c80775c060fa93b633

Author: masatake

.github/workflows/msys2.yml

@@ -0,0 +1,52 @@
+#
+# Derrived from .github/workflows/testing-msys2.yml of universal-ctags/ctags repo
+#
+name: run test cases on MSYS2
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  testing:
+    runs-on: windows-latest
+
+    strategy:
+      matrix:
+        msystem: [MSYS, MINGW32, MINGW64]
+
+    defaults:
+      run:
+        shell: msys2 {0}
+
+    steps:
+    - uses: msys2/setup-msys2@v2
+      with:
+        msystem: ${{ matrix.msystem }}
+        update: true
+        install: git make automake1.16 autoconf libtool
+
+    - if:  matrix.msystem == 'MSYS'
+      run: pacman -S --noconfirm gcc
+
+    - if:  matrix.msystem == 'MINGW32'
+      run: pacman -S --noconfirm mingw-w64-i686-toolchain
+
+    - if:  matrix.msystem == 'MINGW64'
+      run: pacman -S --noconfirm mingw-w64-x86_64-toolchain
+
+    - run: ln -s /usr/bin/automake-1.16 /usr/bin/automake
+    - run: ln -s /usr/bin/aclocal-1.16  /usr/bin/aclocal
+
+    - run: git config --global core.autocrlf input
+      shell: bash
+    - uses: actions/checkout@v2
+
+    - run: ./autogen.sh
+    - run: ./configure --prefix=/usr
+    - run: make V=1
+    - run: make check V=1
+    - run: cat tests/test-suite.log
+      if: failure()

.gitignore

@@ -37,5 +37,6 @@ tests/test-api-tagsFirst
 tests/test-api-tagsFirstPseudoTag
 tests/test-api-tagsOpen
 tests/test-api-tagsSetSortType
+tests/test-fix-large-tags
 tests/test-fix-null-deref
 tests/test-fix-unescaping

NEWS.md

@@ -6,6 +6,17 @@
 - fix potential crashes trigged when passing NULL as `file` parameter
   to the API functions. Provided by rootkea (GitHub account).
 
+- add a new error constant `TagErrnoFileMaybeTooBig` to represent
+  the case that the given tags file is too large for the platform APIs
+  (ftell and fseek) used in libreadtags.
+  See https://github.com/universal-ctags/libreadtags/issues/36 about the
+  background of this change.
+
+- allow the library to read larger (> 2G) tag files on Win32 platform.
+  The tag file size was limited to 2G on the platform because the library
+  used fseek and ftell. In this version, they are replaced with _fseeki64 and
+  _ftelli64.
+
 # Version 0.1.0
 
 - propagate internal errors to caller

configure.ac

@@ -24,8 +24,11 @@ AC_PROG_LIBTOOL
 # 1:0:0
 #    introduced tagsGetErrno() and tagErrno.
 #    rename sortType to tagSortType.
-
-AC_SUBST(LT_VERSION, [1:0:0])
+#
+# 2:0:1
+#    introduced TagErrnoFileMayTooBig.
+#
+AC_SUBST(LT_VERSION, [2:0:1])
 
 AC_ARG_ENABLE([gcov],
 	[AS_HELP_STRING([--enable-gcov],

readtags.c

@@ -32,6 +32,13 @@ typedef struct {
 	char *buffer;
 } vstring;
 
+/* Define readtags' own off_t. */
+#ifdef _WIN32
+typedef long long rt_off_t;
+#else
+typedef off_t rt_off_t;
+#endif
+
 /* Information about current tag file */
 struct sTagFile {
 		/* has the file been opened and this structure initialized? */
@@ -43,17 +50,17 @@ struct sTagFile {
 		/* pointer to file structure */
 	FILE* fp;
 		/* file position of first character of `line' */
-	off_t pos;
+	rt_off_t pos;
 		/* size of tag file in seekable positions */
-	off_t size;
+	rt_off_t size;
 		/* last line read */
 	vstring line;
 		/* name of tag in last line read */
 	vstring name;
 		/* defines tag search state */
 	struct {
 				/* file position of last match for tag */
-			off_t pos;
+			rt_off_t pos;
 				/* name of tag last searched for */
 			char *name;
 				/* length of name for partial matches */
@@ -97,6 +104,32 @@ static const size_t PseudoTagPrefixLength = 2;
 *   FUNCTION DEFINITIONS
 */
 
+static rt_off_t readtags_ftell(FILE *fp)
+{
+	rt_off_t pos;
+
+#ifdef _WIN32
+	pos = _ftelli64(fp);
+#else
+	pos = ftell(fp);
+#endif
+
+	return pos;
+}
+
+static int readtags_fseek(FILE *fp, rt_off_t pos, int whence)
+{
+	int ret;
+
+#ifdef _WIN32
+	ret = _fseeki64(fp, pos, whence);
+#else
+	ret = fseek(fp, pos, whence);
+#endif
+
+	return ret;
+}
+
 /* Converts a hexadecimal digit to its value */
 static int xdigitValue (char digit)
 {
@@ -285,7 +318,7 @@ static int readTagLineRaw (tagFile *const file, int *err)
 		char *const pLastChar = file->line.buffer + file->line.size - 2;
 		char *line;
 
-		file->pos = ftell (file->fp);
+		file->pos = readtags_ftell (file->fp);
 		if (file->pos < 0)
 		{
 			*err = errno;
@@ -312,7 +345,8 @@ static int readTagLineRaw (tagFile *const file, int *err)
 				*err = ENOMEM;
 				result = 0;
 			}
-			if (fseek (file->fp, file->pos, SEEK_SET) < 0)
+
+			if (readtags_fseek (file->fp, file->pos, SEEK_SET) < 0)
 			{
 				*err = errno;
 				result = 0;
@@ -748,7 +782,7 @@ static tagResult gotoFirstLogicalTag (tagFile *const file)
 {
 	fpos_t startOfLine;
 
-	if (fseek(file->fp, 0L, SEEK_SET) == -1)
+	if (readtags_fseek(file->fp, 0, SEEK_SET) == -1)
 	{
 		file->err = errno;
 		return TagFailure;
@@ -804,31 +838,40 @@ static tagFile *initialize (const char *const filePath, tagFileInfo *const info)
 		info->status.error_number = errno;
 		goto file_error;
 	}
-	else
-	{
-		if (fseek (result->fp, 0, SEEK_END) == -1)
-		{
-			info->status.error_number = errno;
-			goto file_error;
-		}
-		result->size = ftell (result->fp);
-		if (result->size == -1)
-		{
-			info->status.error_number = errno;
-			goto file_error;
-		}
-		if (fseek(result->fp, 0L, SEEK_SET) == -1)
-		{
-			info->status.error_number = errno;
-			goto file_error;
-		}
 
-		if (readPseudoTags (result, info) == TagFailure)
-			goto file_error;
+	/* Record the size of the tags file to `size` field of result. */
+	if (readtags_fseek (result->fp, 0, SEEK_END) == -1)
+	{
+		info->status.error_number = errno;
+		goto file_error;
+	}
+	result->size = readtags_ftell (result->fp);
+	if (result->size == -1)
+	{
+		/* fseek() retruns an int value.
+		 * We observed following behavior on Windows;
+		 * if sizeof(int) of the platform is too small for
+		 * representing the size of the tags file, fseek()
+		 * returns -1 and it doesn't set errno.
+		 */
+		info->status.error_number = errno;
+		if (info->status.error_number == 0)
+			info->status.error_number = TagErrnoFileMaybeTooBig;
 
-		info->status.opened = 1;
-		result->initialized = 1;
+		goto file_error;
 	}
+	if (readtags_fseek(result->fp, 0, SEEK_SET) == -1)
+	{
+		info->status.error_number = errno;
+		goto file_error;
+	}
+
+	if (readPseudoTags (result, info) == TagFailure)
+		goto file_error;
+
+	info->status.opened = 1;
+	result->initialized = 1;
+
 	return result;
  mem_error:
 	info->status.error_number = ENOMEM;
@@ -905,9 +948,9 @@ static const char *readFieldValue (
 	return result;
 }
 
-static int readTagLineSeek (tagFile *const file, const off_t pos)
+static int readTagLineSeek (tagFile *const file, const rt_off_t pos)
 {
-	if (fseek (file->fp, pos, SEEK_SET) < 0)
+	if (readtags_fseek (file->fp, pos, SEEK_SET) < 0)
 	{
 		file->err = errno;
 		return 0;
@@ -951,11 +994,11 @@ static tagResult findFirstNonMatchBefore (tagFile *const file)
 #define JUMP_BACK 512
 	int more_lines;
 	int comp;
-	off_t start = file->pos;
-	off_t pos = start;
+	rt_off_t start = file->pos;
+	rt_off_t pos = start;
 	do
 	{
-		if (pos < (off_t) JUMP_BACK)
+		if (pos < (rt_off_t) JUMP_BACK)
 			pos = 0;
 		else
 			pos = pos - JUMP_BACK;
@@ -971,7 +1014,7 @@ static tagResult findFirstMatchBefore (tagFile *const file)
 {
 	tagResult result = TagFailure;
 	int more_lines;
-	off_t start = file->pos;
+	rt_off_t start = file->pos;
 	if (findFirstNonMatchBefore (file) != TagSuccess)
 		return TagFailure;
 	do
@@ -988,10 +1031,10 @@ static tagResult findFirstMatchBefore (tagFile *const file)
 static tagResult findBinary (tagFile *const file)
 {
 	tagResult result = TagFailure;
-	off_t lower_limit = 0;
-	off_t upper_limit = file->size;
-	off_t last_pos = 0;
-	off_t pos = upper_limit / 2;
+	rt_off_t lower_limit = 0;
+	rt_off_t upper_limit = file->size;
+	rt_off_t last_pos = 0;
+	rt_off_t pos = upper_limit / 2;
 	while (result != TagSuccess)
 	{
 		if (! readTagLineSeek (file, pos))
@@ -1083,18 +1126,18 @@ static tagResult find (tagFile *const file, tagEntry *const entry,
 	file->search.nameLength = strlen (name);
 	file->search.partial = (options & TAG_PARTIALMATCH) != 0;
 	file->search.ignorecase = (options & TAG_IGNORECASE) != 0;
-	if (fseek (file->fp, 0, SEEK_END) < 0)
+	if (readtags_fseek (file->fp, 0, SEEK_END) < 0)
 	{
 		file->err = errno;
 		return TagFailure;
 	}
-	file->size = ftell (file->fp);
+	file->size = readtags_ftell (file->fp);
 	if (file->size == -1)
 	{
 		file->err = errno;
 		return TagFailure;
 	}
-	if (fseek(file->fp, 0L, SEEK_SET) == -1)
+	if (readtags_fseek(file->fp, 0, SEEK_SET) == -1)
 	{
 		file->err = errno;
 		return TagFailure;
@@ -1167,7 +1210,7 @@ static tagResult findPseudoTag (tagFile *const file, int rewindBeforeFinding, ta
 
 	if (rewindBeforeFinding)
 	{
-		if (fseek(file->fp, 0L, SEEK_SET) == -1)
+		if (readtags_fseek(file->fp, 0, SEEK_SET) == -1)
 		{
 			file->err = errno;
 			return TagFailure;

readtags.h

@@ -59,6 +59,7 @@ typedef enum {
 										  * (Zero or a positive integer is expected.) */
 	TagErrnoInvalidArgument        = -4, /* Unexpected argument passed to the API
 										  * function */
+	TagErrnoFileMaybeTooBig        = -5, /* Maybe the tags file is too big */
 } tagErrno;
 
 struct sTagFile;
@@ -258,7 +259,7 @@ extern tagResult tagsFindNext (tagFile *const file, tagEntry *const entry);
 
 /*
 *  Does the same as tagsFirst(), but is specialized to pseudo tags.
-*  If tagFileInfo doesn't contain pseudo tags you are interested, read
+*  If tagFileInfo doesn't contain pseudo tags you are interested in, read
 *  them sequentially with this function and tagsNextPseudoTag().
 */
 extern tagResult tagsFirstPseudoTag (tagFile *const file, tagEntry *const entry);

tests/Makefile.am

@@ -9,6 +9,7 @@ TESTS = \
 	\
 	test-fix-unescaping \
 	test-fix-null-deref \
+	test-fix-large-tags \
 	\
 	$(NULL)
 
@@ -23,6 +24,7 @@ check_PROGRAMS = \
 	\
 	test-fix-unescaping \
 	test-fix-null-deref \
+	test-fix-large-tags \
 	\
 	$(NULL)
 
@@ -80,3 +82,6 @@ EXTRA_DIST += unescaping.tags
 test_fix_null_deref = test-fix-null-deref.c
 test_fix_null_deref_DEPENDENCIES = $(DEPS)
 EXTRA_DIST += null-deref.tags
+
+test_fix_large_tags = test-fix-large-tags.c
+test_fix_large_tags_DEPENDENCIES = $(DEPS)