Merge two wordlists

I want to merge two wordlists into one file. All the duplicates must be removed. Each word is separated by a newline. I've searched for this kind of program but I can't find a开发者_开发技巧nything. Am I searching for the right thing? Is there a c/c++ implementation of this?

// read input
std::ifstream in( file_path );
typedef std::set< std::string > wordlist_type;
wordlist_type wordlist;
std::string word;

while ( in >> word ) {
    wordlist.insert( word );
}

// repeat with other files to merge more wordlists

// now output to a new file
std::ofstream out( output_path );
for ( wordlist_type::iterator it = wordlist.begin(); it != wordlist.end(); ++ it ) {
    out << * it << '\n';
}

How big are the files. If you can hold both of them in memory, this is relatively simple using the STL:

std::vector<std::string> v(
        (std::istream_iterator<std::string>( ifile1 )),
        (std::istream_iterator<std::string>()));
v.insert(v.end(),
         std::istream_iterator<std::string>( ifile2 ),
         std::istream_iterator<std::string>());
std::sort( v.begin(), v.end() );
std::copy( v.begin(), std::unique( v.begin(), v.end() ),
           std::ostream_iterator<std::string>( ofile, "\n" ) );

or

std::vector<std::string> v1(
        (std::istream_iterator<std::string>( ifile1 )),
        (std::istream_iterator<std::string>()) );
std::sort( v1.begin(), v1.end() );
v1.erase( std::unique( v1.begin(), v1.end() ), v1.end() );
std::vector<std::string> v2(
        (std::istream_iterator<std::string>( ifile2 )),
        (std::istream_iterator<std::string>()) );
std::sort( v2.begin(), v2.end() );
v2.erase( std::unique( v2.begin(), v2.end() ), v2.end() );
std::set_intersection( v1.begin(), v1.end(),
                       v2.begin(), v2.end(),
                       std::ostream_iterator<std::string>( ofile, "\n" ) );

If they don't fit into memory, you'll probably have to sort each file (using system to invoke your local utility), then do the merge manually:

class FilterDuplicates
{
    std::ostream& myDest;
    std::string myLastOutput;
public:
    Outputter( std::ostream& dest ) : myDest( dest ) {}
    void write( std::string const& word ) const
    {
        if ( word != myLastOutput ) {
            myDest << word;
            myLastOutput = word;
        }
    }
};

ifile1 >> s1;
ifile2 >> s2;
FilterDuplicates out( ofile )
while ( ifile1 && ifile2 ) {
    if ( s1 < s2 ) {
        out.write( s1 );
        ifile1 >> s1;
    } else {
        out.write( s2 );
        ifile2 >> s2;
    }
}
while ( ifile1 ) {
    out.write( s1 );
    ifile1 >> s1;
}
while ( ifile2 ) {
    out.write( s2 );
    ifile2 >> s2;
}

#include <string>
#include <set>
#include <iostream>

int main()
{
    std::set<std::string> s;
    std::string word;
    while (std::cin >> word)
        s.insert(s);
    for (std::set<std::string>::const_iterator i = s.begin(); i != s.end(); ++i)
        std::cout << s << '\n';
}

usage:

cat input1 input2 | program > output

something like this...

std::set<std::string> words;    
std::string word;

while(cin >> word)
  if (words.insert(word).second)
    cout << word;

EDIT: oops, too eager to simplify...

If you have access to unix

cat file1 file2 | sort | uniq > file3