Add word_similarity to pg_trgm contrib module.

feodor · feodor · commit f576b17cd6ba · 2016-03-16T18:59:21.000+03:00
Patch introduces a concept of similarity over string and just a word from
another string.

Version of extension is not changed because 1.2 was already introduced in 9.6
release cycle, so, there wasn't a public version.

Author: Alexander Korotkov, Artur Zakirov
diff --git a/contrib/pg_trgm/Makefile b/contrib/pg_trgm/Makefile
@@ -7,7 +7,7 @@ EXTENSION = pg_trgm
 DATA = pg_trgm--1.2.sql pg_trgm--1.0--1.1.sql pg_trgm--1.1--1.2.sql pg_trgm--unpackaged--1.0.sql
 PGFILEDESC = "pg_trgm - trigram matching"
 
-REGRESS = pg_trgm
+REGRESS = pg_trgm pg_word_trgm
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
diff --git a/contrib/pg_trgm/expected/pg_trgm.out b/contrib/pg_trgm/expected/pg_trgm.out
@@ -59,7 +59,7 @@ select similarity('---', '####---');
           0
 (1 row)
 
-CREATE TABLE test_trgm(t text);
+CREATE TABLE test_trgm(t text COLLATE "C");
 \copy test_trgm from 'data/trgm.data'
 select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu0988' order by sml desc, t;
       t      |   sml    
@@ -3467,7 +3467,7 @@ select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu198
  qwertyu0988 | 0.333333
 (1 row)
 
-create table test2(t text);
+create table test2(t text COLLATE "C");
 insert into test2 values ('abcdef');
 insert into test2 values ('quark');
 insert into test2 values ('  z foo bar');
diff --git a/contrib/pg_trgm/pg_trgm--1.1--1.2.sql b/contrib/pg_trgm/pg_trgm--1.1--1.2.sql
@@ -3,10 +3,72 @@
 -- complain if script is sourced in psql, rather than via ALTER EXTENSION
 \echo Use "ALTER EXTENSION pg_trgm UPDATE TO '1.2'" to load this file. \quit
 
+CREATE FUNCTION word_similarity(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION word_similarity_op(text,text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT STABLE;  -- stable because depends on pg_trgm.word_similarity_threshold
+
+CREATE FUNCTION word_similarity_commutator_op(text,text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT STABLE;  -- stable because depends on pg_trgm.word_similarity_threshold
+
+CREATE FUNCTION word_similarity_dist_op(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION word_similarity_dist_commutator_op(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE OPERATOR <% (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_op,
+        COMMUTATOR = '%>',
+        RESTRICT = contsel,
+        JOIN = contjoinsel
+);
+
+CREATE OPERATOR %> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_commutator_op,
+        COMMUTATOR = '<%',
+        RESTRICT = contsel,
+        JOIN = contjoinsel
+);
+
+CREATE OPERATOR <<-> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_dist_op,
+        COMMUTATOR = '<->>'
+);
+
+CREATE OPERATOR <->> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_dist_commutator_op,
+        COMMUTATOR = '<<->'
+);
+
 CREATE FUNCTION gin_trgm_triconsistent(internal, int2, text, int4, internal, internal, internal)
 RETURNS "char"
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
+ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
+        OPERATOR        7       %> (text, text),
+        OPERATOR        8       <->> (text, text) FOR ORDER BY pg_catalog.float_ops;
+
 ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD
-        FUNCTION        6    (text, text)   gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);
+        OPERATOR        7       %> (text, text),
+        FUNCTION        6      (text, text)   gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);
diff --git a/contrib/pg_trgm/pg_trgm--1.2.sql b/contrib/pg_trgm/pg_trgm--1.2.sql
@@ -39,6 +39,39 @@ CREATE OPERATOR % (
         JOIN = contjoinsel
 );
 
+CREATE FUNCTION word_similarity(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION word_similarity_op(text,text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT STABLE;  -- stable because depends on pg_trgm.word_similarity_threshold
+
+CREATE FUNCTION word_similarity_commutator_op(text,text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT STABLE;  -- stable because depends on pg_trgm.word_similarity_threshold
+
+CREATE OPERATOR <% (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_op,
+        COMMUTATOR = '%>',
+        RESTRICT = contsel,
+        JOIN = contjoinsel
+);
+
+CREATE OPERATOR %> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_commutator_op,
+        COMMUTATOR = '<%',
+        RESTRICT = contsel,
+        JOIN = contjoinsel
+);
+
 CREATE FUNCTION similarity_dist(text,text)
 RETURNS float4
 AS 'MODULE_PATHNAME'
@@ -51,6 +84,30 @@ CREATE OPERATOR <-> (
         COMMUTATOR = '<->'
 );
 
+CREATE FUNCTION word_similarity_dist_op(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION word_similarity_dist_commutator_op(text,text)
+RETURNS float4
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT IMMUTABLE;
+
+CREATE OPERATOR <<-> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_dist_op,
+        COMMUTATOR = '<->>'
+);
+
+CREATE OPERATOR <->> (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = word_similarity_dist_commutator_op,
+        COMMUTATOR = '<<->'
+);
+
 -- gist key
 CREATE FUNCTION gtrgm_in(cstring)
 RETURNS gtrgm
@@ -140,6 +197,12 @@ ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
         OPERATOR        5       pg_catalog.~ (text, text),
         OPERATOR        6       pg_catalog.~* (text, text);
 
+-- Add operators that are new in 9.6 (pg_trgm 1.2).
+
+ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
+        OPERATOR        7       %> (text, text),
+        OPERATOR        8       <->> (text, text) FOR ORDER BY pg_catalog.float_ops;
+
 -- support functions for gin
 CREATE FUNCTION gin_extract_value_trgm(text, internal)
 RETURNS internal
@@ -187,4 +250,5 @@ AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
 ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD
+        OPERATOR        7       %> (text, text),
         FUNCTION        6      (text,text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);
diff --git a/contrib/pg_trgm/sql/pg_trgm.sql b/contrib/pg_trgm/sql/pg_trgm.sql
@@ -13,7 +13,7 @@ select similarity('wow',' WOW ');
 
 select similarity('---', '####---');
 
-CREATE TABLE test_trgm(t text);
+CREATE TABLE test_trgm(t text COLLATE "C");
 
 \copy test_trgm from 'data/trgm.data'
 
@@ -40,7 +40,7 @@ select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu098
 select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu0988' order by sml desc, t;
 select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
 
-create table test2(t text);
+create table test2(t text COLLATE "C");
 insert into test2 values ('abcdef');
 insert into test2 values ('quark');
 insert into test2 values ('  z foo bar');
diff --git a/contrib/pg_trgm/trgm.h b/contrib/pg_trgm/trgm.h
@@ -26,13 +26,14 @@
 #define DIVUNION
 
 /* operator strategy numbers */
-#define SimilarityStrategyNumber	1
-#define DistanceStrategyNumber		2
-#define LikeStrategyNumber			3
-#define ILikeStrategyNumber			4
-#define RegExpStrategyNumber		5
-#define RegExpICaseStrategyNumber	6
-
+#define SimilarityStrategyNumber		1
+#define DistanceStrategyNumber			2
+#define LikeStrategyNumber				3
+#define ILikeStrategyNumber				4
+#define RegExpStrategyNumber			5
+#define RegExpICaseStrategyNumber		6
+#define WordSimilarityStrategyNumber	7
+#define WordDistanceStrategyNumber		8
 
 typedef char trgm[3];
 
@@ -103,15 +104,28 @@ typedef char *BITVECP;
 #define GETARR(x)		( (trgm*)( (char*)x+TRGMHDRSIZE ) )
 #define ARRNELEM(x) ( ( VARSIZE(x) - TRGMHDRSIZE )/sizeof(trgm) )
 
+/*
+ * If DIVUNION is defined then similarity formula is:
+ * count / (len1 + len2 - count)
+ * else if DIVUNION is not defined then similarity formula is:
+ * count / max(len1, len2)
+ */
+#ifdef DIVUNION
+#define CALCSML(count, len1, len2) ((float4) (count)) / ((float4) ((len1) + (len2) - (count)))
+#else
+#define CALCSML(count, len1, len2) ((float4) (count)) / ((float4) (((len1) > (len2)) ? (len1) : (len2)))
+#endif
+
 typedef struct TrgmPackedGraph TrgmPackedGraph;
 
 extern double similarity_threshold;
+extern double word_similarity_threshold;
 
 extern uint32 trgm2int(trgm *ptr);
 extern void compact_trigram(trgm *tptr, char *str, int bytelen);
 extern TRGM *generate_trgm(char *str, int slen);
 extern TRGM *generate_wildcard_trgm(const char *str, int slen);
-extern float4 cnt_sml(TRGM *trg1, TRGM *trg2);
+extern float4 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact);
 extern bool trgm_contained_by(TRGM *trg1, TRGM *trg2);
 extern bool *trgm_presence_map(TRGM *query, TRGM *key);
 extern TRGM *createTrgmNFA(text *text_re, Oid collation,
diff --git a/contrib/pg_trgm/trgm_gin.c b/contrib/pg_trgm/trgm_gin.c
@@ -89,6 +89,7 @@ gin_extract_query_trgm(PG_FUNCTION_ARGS)
 	switch (strategy)
 	{
 		case SimilarityStrategyNumber:
+		case WordSimilarityStrategyNumber:
 			trg = generate_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
 			break;
 		case ILikeStrategyNumber:
@@ -176,13 +177,18 @@ gin_trgm_consistent(PG_FUNCTION_ARGS)
 	bool		res;
 	int32		i,
 				ntrue;
+	double		nlimit;
 
 	/* All cases served by this function are inexact */
 	*recheck = true;
 
 	switch (strategy)
 	{
 		case SimilarityStrategyNumber:
+		case WordSimilarityStrategyNumber:
+			nlimit = (strategy == SimilarityStrategyNumber) ?
+				similarity_threshold : word_similarity_threshold;
+
 			/* Count the matches */
 			ntrue = 0;
 			for (i = 0; i < nkeys; i++)
@@ -207,8 +213,7 @@ gin_trgm_consistent(PG_FUNCTION_ARGS)
 			 * So, independly on DIVUNION the upper bound formula is the same.
 			 */
 			res = (nkeys == 0) ? false :
-				((((((float4) ntrue) / ((float4) nkeys))) >= similarity_threshold)
-					? true : false);
+				(((((float4) ntrue) / ((float4) nkeys))) >= nlimit);
 			break;
 		case ILikeStrategyNumber:
 #ifndef IGNORECASE
@@ -270,10 +275,15 @@ gin_trgm_triconsistent(PG_FUNCTION_ARGS)
 	int32		i,
 				ntrue;
 	bool	   *boolcheck;
+	double		nlimit;
 
 	switch (strategy)
 	{
 		case SimilarityStrategyNumber:
+		case WordSimilarityStrategyNumber:
+			nlimit = (strategy == SimilarityStrategyNumber) ?
+				similarity_threshold : word_similarity_threshold;
+
 			/* Count the matches */
 			ntrue = 0;
 			for (i = 0; i < nkeys; i++)
@@ -285,9 +295,9 @@ gin_trgm_triconsistent(PG_FUNCTION_ARGS)
 			/*
 			 * See comment in gin_trgm_consistent() about * upper bound formula
 			 */
-			res = (nkeys == 0) ? GIN_FALSE :
-				(((((float4) ntrue) / ((float4) nkeys)) >= similarity_threshold)
-					? GIN_MAYBE : GIN_FALSE);
+			res = (nkeys == 0)
+				? GIN_FALSE : (((((float4) ntrue) / ((float4) nkeys)) >= nlimit)
+							? GIN_MAYBE : GIN_FALSE);
 			break;
 		case ILikeStrategyNumber:
 #ifndef IGNORECASE
diff --git a/contrib/pg_trgm/trgm_gist.c b/contrib/pg_trgm/trgm_gist.c
diff --git a/contrib/pg_trgm/trgm_op.c b/contrib/pg_trgm/trgm_op.c
diff --git a/doc/src/sgml/pgtrgm.sgml b/doc/src/sgml/pgtrgm.sgml
