Snowball multibyte. It's a pity, but snowball sources is very diferent for multibyte and

singlebyte encodings, so we should have snowball for every encodings.

I hope that finalize multibyte support work in tsearch2, but testing is needed...

Author: feodor

contrib/tsearch2/Makefile

@@ -1,4 +1,4 @@
-# $PostgreSQL: pgsql/contrib/tsearch2/Makefile,v 1.12 2005/11/21 12:27:57 teodor Exp $
+# $PostgreSQL: pgsql/contrib/tsearch2/Makefile,v 1.13 2006/01/27 16:32:31 teodor Exp $
 
 MODULE_big = tsearch2
 OBJS = dict_ex.o dict.o snmap.o stopword.o common.o prs_dcfg.o \
@@ -16,7 +16,7 @@ OBJS	+= $(SUBDIROBJS)
 
 PG_CPPFLAGS = -I$(srcdir)/snowball -I$(srcdir)/ispell -I$(srcdir)/wordparser
 
-DATA = stopword/english.stop stopword/russian.stop
+DATA = stopword/english.stop stopword/russian.stop stopword/russian.stop.utf8
 DATA_built = tsearch2.sql untsearch2.sql
 DOCS = README.tsearch2
 REGRESS = tsearch2

contrib/tsearch2/dict_snowball.c

@@ -10,6 +10,7 @@
 #include "snowball/header.h"
 #include "snowball/english_stem.h"
 #include "snowball/russian_stem.h"
+#include "snowball/russian_stem_UTF8.h"
 #include "ts_locale.h"
 
 typedef struct
@@ -23,8 +24,11 @@ typedef struct
 PG_FUNCTION_INFO_V1(snb_en_init);
 Datum		snb_en_init(PG_FUNCTION_ARGS);
 
-PG_FUNCTION_INFO_V1(snb_ru_init);
-Datum		snb_ru_init(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(snb_ru_init_koi8);
+Datum		snb_ru_init_koi8(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(snb_ru_init_utf8);
+Datum		snb_ru_init_utf8(PG_FUNCTION_ARGS);
 
 PG_FUNCTION_INFO_V1(snb_lexize);
 Datum		snb_lexize(PG_FUNCTION_ARGS);
@@ -64,7 +68,7 @@ snb_en_init(PG_FUNCTION_ARGS)
 }
 
 Datum
-snb_ru_init(PG_FUNCTION_ARGS)
+snb_ru_init_koi8(PG_FUNCTION_ARGS)
 {
 	DictSnowball *d = (DictSnowball *) malloc(sizeof(DictSnowball));
 
@@ -97,6 +101,40 @@ snb_ru_init(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(d);
 }
 
+Datum
+snb_ru_init_utf8(PG_FUNCTION_ARGS)
+{
+	DictSnowball *d = (DictSnowball *) malloc(sizeof(DictSnowball));
+
+	if (!d)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory")));
+	memset(d, 0, sizeof(DictSnowball));
+	d->stoplist.wordop = lowerstr;
+
+	if (!PG_ARGISNULL(0) && PG_GETARG_POINTER(0) != NULL)
+	{
+		text	   *in = PG_GETARG_TEXT_P(0);
+
+		readstoplist(in, &(d->stoplist));
+		sortstoplist(&(d->stoplist));
+		PG_FREE_IF_COPY(in, 0);
+	}
+
+	d->z = russian_UTF_8_create_env();
+	if (!d->z)
+	{
+		freestoplist(&(d->stoplist));
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory")));
+	}
+	d->stem = russian_UTF_8_stem;
+
+	PG_RETURN_POINTER(d);
+}
+
 Datum
 snb_lexize(PG_FUNCTION_ARGS)
 {

contrib/tsearch2/expected/tsearch2.out

@@ -4,21 +4,21 @@
 --
 \set ECHO none
 psql:tsearch2.sql:13: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_dict_pkey" for table "pg_ts_dict"
-psql:tsearch2.sql:145: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_parser_pkey" for table "pg_ts_parser"
-psql:tsearch2.sql:244: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_cfg_pkey" for table "pg_ts_cfg"
-psql:tsearch2.sql:251: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_cfgmap_pkey" for table "pg_ts_cfgmap"
-psql:tsearch2.sql:337: NOTICE:  type "tsvector" is not yet defined
+psql:tsearch2.sql:158: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_parser_pkey" for table "pg_ts_parser"
+psql:tsearch2.sql:257: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_cfg_pkey" for table "pg_ts_cfg"
+psql:tsearch2.sql:264: NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "pg_ts_cfgmap_pkey" for table "pg_ts_cfgmap"
+psql:tsearch2.sql:370: NOTICE:  type "tsvector" is not yet defined
 DETAIL:  Creating a shell type definition.
-psql:tsearch2.sql:342: NOTICE:  argument type tsvector is only a shell
-psql:tsearch2.sql:396: NOTICE:  type "tsquery" is not yet defined
+psql:tsearch2.sql:375: NOTICE:  argument type tsvector is only a shell
+psql:tsearch2.sql:429: NOTICE:  type "tsquery" is not yet defined
 DETAIL:  Creating a shell type definition.
-psql:tsearch2.sql:401: NOTICE:  argument type tsquery is only a shell
-psql:tsearch2.sql:559: NOTICE:  type "gtsvector" is not yet defined
+psql:tsearch2.sql:434: NOTICE:  argument type tsquery is only a shell
+psql:tsearch2.sql:592: NOTICE:  type "gtsvector" is not yet defined
 DETAIL:  Creating a shell type definition.
-psql:tsearch2.sql:564: NOTICE:  argument type gtsvector is only a shell
-psql:tsearch2.sql:1054: NOTICE:  type "gtsq" is not yet defined
+psql:tsearch2.sql:597: NOTICE:  argument type gtsvector is only a shell
+psql:tsearch2.sql:1087: NOTICE:  type "gtsq" is not yet defined
 DETAIL:  Creating a shell type definition.
-psql:tsearch2.sql:1059: NOTICE:  argument type gtsq is only a shell
+psql:tsearch2.sql:1092: NOTICE:  argument type gtsq is only a shell
 --tsvector
 SELECT '1'::tsvector;
  tsvector 

contrib/tsearch2/snowball/Makefile

@@ -1,6 +1,6 @@
-# $PostgreSQL: pgsql/contrib/tsearch2/snowball/Makefile,v 1.8 2005/10/18 01:30:48 tgl Exp $
+# $PostgreSQL: pgsql/contrib/tsearch2/snowball/Makefile,v 1.9 2006/01/27 16:32:31 teodor Exp $
 
-SUBOBJS = english_stem.o api.o russian_stem.o utilities.o
+SUBOBJS = english_stem.o api.o russian_stem.o russian_stem_UTF8.o utilities.o
 
 EXTRA_CLEAN = SUBSYS.o $(SUBOBJS)