Join [one word per row] to rows of phrases with [multiple words per row]

Please excuse the length of the question. I included a test script to demo the situation and my best attempt at a solution.

There are two tables:

1. test_WORDS = Words extracted in order from several sources. The OBJ_FK column is the ID of the source. WORD_ID is an identifier for the word itself that is unique within the source. Each row contains one word.
2. test_PHRASE = a list of phrases to be searched for in test_WORDS. The PHRASE_TEXT column is a space separated phrase like 'foo bar' (see below) so that each row contains multiple words.

Requirement: Return the first word from test_WORDS that is the start of a matching a phrase from test_PHRASE.

I would prefer something set based to avoid RBAR approach below. Also my solution is limited to 5 word phrases. I need to support up to 20 word phrases. Is it possible to match the words from a row in test_PHRASE to contiguous rows in the test_WORD without cursors?

After breaking the phrase words out into a temporary table, the problem boils down to matching portions of two sets together in row order.

-- Create test data
CREATE TABLE [dbo].[test_WORDS](
    [OBJ_FK] [bigint] NOT NULL,             --FK to the source object
    [WORD_ID] [int] NOT NULL,               --The word order in the source object
    [WORD_TEXT] [nvarchar](50) NOT NULL,
     CONSTRAINT [PK_test_WORDS] PRIMARY KEY CLUSTERED 
    (
        [OBJ_FK] ASC,
        [WORD_ID] ASC
    )
) ON [PRIMARY]    
GO

CREATE TABLE [dbo].[test_PHRASE](
    [ID] [int],     --PHRASE ID
    [PHRASE_TEXT] [nvarchar](150) NOT NULL  --Space-separated phrase
     CONSTRAINT [PK_test_PHRASE] PRIMARY KEY CLUSTERED 
    (
        [ID] ASC
    )
)
GO
INSERT INTO dbo.test_WORDS
SELECT 1,1,'aaa' UNION ALL
SELECT 1,2,'bbb' UNION ALL
SELECT 1,3,'ccc' UNION ALL
SELECT 1,4,'ddd' UNION ALL
SELECT 1,5,'eee' UNION ALL
SELECT 1,6,'fff' UNION ALL
SELECT 1,7,'ggg' UNION ALL
SELECT 1,8,'hhh' UNION ALL
SELECT 2,1,'zzz' UNION ALL
SELECT 2,2,'yyy' UNION ALL
SELECT 2,3,'xxx' UNION ALL
SELECT 2,4,'www'

INSERT INTO dbo.test_PHRASE
SELECT 1, 'bbb ccc ddd' UNION ALL --should match 
SELECT 2, 'ddd eee fff' UNION ALL --should match 
SELECT 3, 'xxx xxx xxx' UNION ALL --should NOT match 
SELECT 4, 'zzz yyy xxx' UNION ALL --should match 
SELECT 5, 'xxx www ppp' UNION ALL --should NOT match 
SELECT 6, 'zzz yyy xxx www'    --should match 

-- Create variables
DECLARE @maxRow AS INTEGER
DECLARE @currentRow AS INTEGER
DECLARE @phraseSubsetTable AS TABLE(
    [ROW] int IDENTITY(1,1) NOT NULL,
    [ID] int NOT NULL,      --PHRASE ID
    [PH开发者_如何学GoRASE_TEXT] nvarchar(150) NOT NULL
)
--used to split the phrase into words
--note:  No permissions to sys.dm_fts_parser
DECLARE @WordList table
(
    ID int,
    WORD nvarchar(50)
)
--Records to be returned to caller
DECLARE @returnTable AS TABLE(
    OBJECT_FK INT NOT NULL,
    WORD_ID INT NOT NULL,
    PHRASE_ID INT NOT NULL
)
DECLARE @phrase AS NVARCHAR(150)
DECLARE @phraseID AS INTEGER

-- Get subset of phrases to simulate a join that would occur in production
INSERT INTO @phraseSubsetTable 
SELECT ID, PHRASE_TEXT 
FROM dbo.test_PHRASE
--represent subset of phrases caused by join in production
WHERE ID IN (2,3,4)

-- Loop each phrase in the subset, split into rows of words and return matches to the test_WORDS table
SET @maxRow = @@ROWCOUNT
SET @currentRow = 1
WHILE @currentRow <= @maxRow
BEGIN
    SELECT @phrase=PHRASE_TEXT, @phraseID=ID FROM @phraseSubsetTable WHERE row = @currentRow

    --clear previous phrase that was split into rows
    DELETE FROM @WordList

    --Recursive Function with CTE to create recordset of words, one per row
    ;WITH Pieces(pn, start, stop) AS (
      SELECT 1, 1, CHARINDEX(' ', @phrase)
      UNION ALL
      SELECT pn + 1, stop + 1, CHARINDEX(' ', @phrase, stop + 1)
      FROM Pieces
      WHERE stop > 0)
    --Create the List of words with the CTE above
    insert into @WordList
    SELECT pn,
      SUBSTRING(@phrase, start, CASE WHEN stop > 0 THEN stop-start ELSE 1056 END) AS WORD
    FROM Pieces

    DECLARE @wordCt as int
    select @wordCt=count(ID) from @WordList;

    -- Do the actual query using a CTE with a rownumber that repeats for every SOURCE OBJECT
;WITH WordOrder_CTE AS (
SELECT OBJ_FK, WORD_ID, WORD_TEXT,
    ROW_NUMBER() OVER (Partition BY OBJ_FK ORDER BY WORD_ID) AS rownum 
FROM test_WORDS)
--CREATE a flattened record of the first word in the phrase and join it to the rest of the words.
INSERT INTO @returnTable  
SELECT r1.OBJ_FK, r1.WORD_ID, @phraseID AS PHRASE_ID
FROM WordOrder_CTE r1 
INNER JOIN @WordList w1 ON r1.WORD_TEXT = w1.WORD and w1.ID=1
LEFT JOIN WordOrder_CTE r2 
        ON r1.rownum = r2.rownum - 1 and r1.OBJ_FK = r2.OBJ_FK
            LEFT JOIN @WordList w2 ON r2.WORD_TEXT = w2.WORD and w2.ID=2
LEFT JOIN WordOrder_CTE r3 
        ON r1.rownum = r3.rownum - 2 and r1.OBJ_FK = r3.OBJ_FK 
            LEFT JOIN @WordList w3 ON r3.WORD_TEXT = w3.WORD and w3.ID=3
LEFT JOIN WordOrder_CTE r4
        ON r1.rownum = r4.rownum - 3 and r1.OBJ_FK = r4.OBJ_FK
            LEFT JOIN @WordList w4 ON r4.WORD_TEXT = w4.WORD and w4.ID=4
LEFT JOIN WordOrder_CTE r5
        ON r1.rownum = r5.rownum - 4 and r1.OBJ_FK = r5.OBJ_FK
            LEFT JOIN @WordList w5 ON r5.WORD_TEXT = w5.WORD and w5.ID=5

WHERE   (@wordCt < 2 OR w2.ID is not null) and
        (@wordCt < 3 OR w3.ID is not null) and
        (@wordCt < 4 OR w4.ID is not null) and
        (@wordCt < 5 OR w5.ID is not null)

    --loop
    SET @currentRow = @currentRow+1
END 

--Return the first words of each matching phrase
SELECT  OBJECT_FK, WORD_ID, PHRASE_ID FROM @returnTable

GO

--Clean up
DROP TABLE [dbo].[test_WORDS]
DROP TABLE [dbo].[test_PHRASE]

Edited solution:

This is an edit of the correct solution provided below to account for non-contiguous word IDs. Hope this helps someone as much as it did me.

;WITH
numberedwords AS (
  SELECT
    OBJ_FK,
    WORD_ID,
    WORD_TEXT,
    rowcnt =  ROW_NUMBER() OVER
      (PARTITION BY OBJ_FK ORDER BY WORD_ID DESC),
    totalInSrc = COUNT(WORD_ID) OVER (PARTITION BY OBJ_FK)
  FROM dbo.test_WORDS
),
phrasedwords AS (
  SELECT
    nw1.OBJ_FK,
    nw1.WORD_ID,
    nw1.WORD_TEXT,
    PHRASE_TEXT = RTRIM((
      SELECT [text()] = nw2.WORD_TEXT + ' '
      FROM numberedwords nw2
      WHERE nw1.OBJ_FK = nw2.OBJ_FK
         AND nw2.rowcnt BETWEEN nw1.rowcnt AND nw1.totalInSrc
      ORDER BY nw2.OBJ_FK, nw2.WORD_ID
      FOR XML PATH ('')
    ))
  FROM numberedwords nw1
  GROUP BY nw1.OBJ_FK, nw1.WORD_ID, nw1.WORD_TEXT, nw1.rowcnt, nw1.totalInSrc
)
SELECT *
FROM phrasedwords pw
  INNER JOIN test_PHRASE tp
    ON LEFT(pw.PHRASE_TEXT, LEN(tp.PHRASE_TEXT)) = tp.PHRASE_TEXT
ORDER BY pw.OBJ_FK, pw.WORD_ID

Note: The final query I used in production uses indexed temp tables instead of CTEs. I also limited the length of the PHRASE_TEXT column to my needs. With these improvements, I was able to reduce my query time from over 3 minutes to 3 seconds!

Here's a solution that uses a different approach: instead of splitting the phrases into words it combines the words into phrases.

_{Edited: changed the rowcnt expression to using COUNT(*) OVER …, as suggested by @ErikE in the comments.}

;WITH
numberedwords AS (
  SELECT
    OBJ_FK,
    WORD_ID,
    WORD_TEXT,
    rowcnt =  COUNT(*) OVER (PARTITION BY OBJ_FK)
  FROM dbo.test_WORDS
),
phrasedwords AS (
  SELECT
    nw1.OBJ_FK,
    nw1.WORD_ID,
    nw1.WORD_TEXT,
    PHRASE_TEXT = RTRIM((
      SELECT [text()] = nw2.WORD_TEXT + ' '
      FROM numberedwords nw2
      WHERE nw1.OBJ_FK = nw2.OBJ_FK
        AND nw2.WORD_ID BETWEEN nw1.WORD_ID AND nw1.rowcnt
      ORDER BY nw2.OBJ_FK, nw2.WORD_ID
      FOR XML PATH ('')
    ))
  FROM numberedwords nw1
  GROUP BY nw1.OBJ_FK, nw1.WORD_ID, nw1.WORD_TEXT, nw1.rowcnt
)
SELECT *
FROM phrasedwords pw
  INNER JOIN test_PHRASE tp
    ON LEFT(pw.PHRASE_TEXT, LEN(tp.PHRASE_TEXT)) = tp.PHRASE_TEXT
ORDER BY pw.OBJ_FK, pw.WORD_ID

Using a Split function should work.

Split Function

CREATE FUNCTION dbo.Split
(
    @RowData nvarchar(2000),
    @SplitOn nvarchar(5)
)  
RETURNS @RtnValue table 
(
    Id int identity(1,1),
    Data nvarchar(100)
) 
AS  
BEGIN 
    Declare @Cnt int
    Set @Cnt = 1

    While (Charindex(@SplitOn,@RowData)>0)
    Begin
        Insert Into @RtnValue (data)
        Select 
            Data = ltrim(rtrim(Substring(@RowData,1,Charindex(@SplitOn,@RowData)-1)))

        Set @RowData = Substring(@RowData,Charindex(@SplitOn,@RowData)+1,len(@RowData))
        Set @Cnt = @Cnt + 1
    End

    Insert Into @RtnValue (data)
    Select Data = ltrim(rtrim(@RowData))

    Return
END

SQL Statement

SELECT  DISTINCT p.*
FROM    dbo.test_PHRASE p
        LEFT OUTER JOIN (
          SELECT  p.ID
          FROM    dbo.test_PHRASE p
                  CROSS APPLY dbo.Split(p.PHRASE_TEXT, ' ') sp
                  LEFT OUTER JOIN dbo.test_WORDS w ON w.WORD_TEXT = sp.Data 
          WHERE   w.OBJ_FK IS NULL
        ) ignore ON ignore.ID = p.ID
WHERE   ignore.ID IS NULL        

This performs a little better than other solutions given. if you don't need WORD_ID, just WORD_TEXT, you can remove a whole column. I know this was over a year ago, but I wonder if you can get 3 seconds down to 30 ms? :)

If this query seems good, then my biggest speed advice is to put the entire phrases into a separate table (using your example data, it would have only 2 rows with phrases of length 8 words and 4 words).

SELECT
   W.OBJ_FK,
   X.Phrase,
   P.*,
   Left(P.PHRASE_TEXT, 
      IsNull(NullIf(CharIndex(' ', P.PHRASE_TEXT), 0) - 1, 2147483647)
   ) WORD_TEXT,
   Len(Left(X.Phrase, PatIndex('%' + P.PHRASE_TEXT + '%', ' ' + X.Phrase) - 1))
      - Len(Replace(
         Left(X.Phrase, PatIndex('%' + P.PHRASE_TEXT + '%', X.Phrase) - 1), ' ', '')
      )
      WORD_ID
FROM
   (SELECT DISTINCT OBJ_FK FROM dbo.test_WORDS) W
   CROSS APPLY (
      SELECT RTrim((SELECT WORD_TEXT + ' '
      FROM dbo.test_WORDS W2
      WHERE W.OBJ_FK = W2.OBJ_FK
      ORDER BY W2.WORD_ID
      FOR XML PATH (''))) Phrase
   ) X
   INNER JOIN dbo.test_PHRASE P
      ON X.Phrase LIKE '%' + P.PHRASE_TEXT + '%';

Here's another version for curiosity's sake. It doesn't perform quite as well.

WITH Calc AS (
   SELECT
      P.ID,
      P.PHRASE_TEXT,
      W.OBJ_FK,
      W.WORD_ID StartID,
      W.WORD_TEXT StartText,
      W.WORD_ID,
      Len(W.WORD_TEXT) + 2 NextPos,
      Convert(varchar(150), W.WORD_TEXT) MatchingPhrase
   FROM
      dbo.test_PHRASE P
      INNER JOIN dbo.test_WORDS W
         ON P.PHRASE_TEXT + ' ' LIKE W.WORD_TEXT + ' %'
   UNION ALL
   SELECT
      C.ID,
      C.PHRASE_TEXT,
      C.OBJ_FK,
      C.StartID,
      C.StartText,
      W.WORD_ID,
      C.NextPos + Len(W.WORD_TEXT) + 1,
      Convert(varchar(150), C.MatchingPhrase + Coalesce(' ' + W.WORD_TEXT, ''))
   FROM
      Calc C
      INNER JOIN dbo.test_WORDS W
         ON C.OBJ_FK = W.OBJ_FK
         AND C.WORD_ID + 1 = W.WORD_ID
         AND Substring(C.PHRASE_TEXT, C.NextPos, 2147483647) + ' ' LIKE W.WORD_TEXT + ' %'
)
SELECT C.OBJ_FK, C.PHRASE_TEXT, C.StartID, C.StartText, C.ID
FROM Calc C
WHERE C.PHRASE_TEXT = C.MatchingPhrase;