#include "stdafx.h"
#include <iostream>
#include <string>
#include <vector>

using namespace std;

namespace boko
{
	
	class tokenizer
	{
	public:
		vector<string> m_strStream;
		typedef vector<string>::iterator iterator;
		tokenizer(string& strStream,const char* szFilter)
		{
			string strTmp;
			for(string::iterator it = strStream.begin(); it != strStream.end();it++)
			{
				if(strchr(szFilter,*it))
				{
					if(strTmp != "")
					{
						m_strStream.push_back(strTmp);
						strTmp.clear();
					}
					else
					{
						continue;
					}
				}
				else
				{
					strTmp.push_back(*it);
				}
			}
			
			if(strTmp != "")
			{
				m_strStream.push_back(strTmp);
			}
		}
		iterator begin()
		{
			return m_strStream.begin();
		}
		iterator end()
		{
			return m_strStream.end();
		}
	};
};

#include <iostream>

namespace boko
{
	class tokenizer
	{
		typedef char** iterator;
		
	public:
		tokenizer(std::string& token,std::string& mask)
		{
			m_strToken=token;
			m_strMask=mask;
			m_nCount=0;
			m_nAllocated=10;
			m_pToken=new char* [m_nAllocated];
			
			tokenit();
		}
		~tokenizer()
		{
			delete [] m_pToken;
		}
		
	public:
		iterator begin()
		{
			return (iterator)m_pToken;
		}
		
		iterator end()
		{
			return (iterator)(m_pToken+m_nCount);
		}
		
	protected:
		void tokenit()
		{
			bool bmask=false;
			for(int i=0;i<m_strToken.size();i++)
			{
				if(ismask(m_strToken[i]))
				{
					m_strToken[i]='\0';
					bmask=true;
				}
				else
				{
					if(bmask==true)
					{
						m_pToken[m_nCount++]=&m_strToken[i];
						if(m_nCount>=m_nAllocated)
						{
							m_nAllocated+=10;
							m_pToken=(char **)realloc(m_pToken,m_nAllocated*sizeof(char*));
						}
					}
					bmask=false;
				}
			}
		}
		
		bool ismask(char c)
		{
			for(int i=0;i<m_strMask.size();i++)
				if(c==m_strMask[i])
					return true;
				return false;
		}
		
	private:
		std::string m_strToken;
		std::string m_strMask;
		char** m_pToken;
		int m_nCount;
		int m_nAllocated;
	};
}

void test_tokenizer()
{
	std::string str = ";;Hello|world||-foo--bar;yow;baz|";
	
	boko::tokenizer tokens(str, "-;|");
	
	for (boko::tokenizer::iterator it = tokens.begin();
	it != tokens.end(); ++it)
		std::cout << "<" << *it << "> ";
}

void main()
{
	test_tokenizer();
}

class tokenizer
{
public:
	tokenizer(char* str,char* mask)
	{
		dStrlen = strlen(str);
		m_Array = new char*[dStrlen];/*随便申请一个足够大小的内存*/ 
		m_point = m_Array; 
		sum = 0;
		for (int i= 0;i<dStrlen;i++)
		{
			bool lastfind;
			char t[2] = {str[i],0};
			if (strstr(mask,t))
			{
				str[i] = 0;
				lastfind = true;
			}
			else
			{
				if (lastfind)
				{
					*m_point++ = &str[i];
					lastfind = false;
					sum++;
				}
			}
		}
	}

	~tokenizer(){delete[] m_Array;}

	char** begin()
	{
		return m_Array;
	}
	char** end()
	{
		return &m_Array[sum];
	}
private:
	int dStrlen;
	char** m_Array;
	char** m_point;
	int sum;
};

void test_tokenizer()
{
	char str[] = ";;Hello|world||-foo--bar;yow;baz|";

	tokenizer tokens(str, "-;|");

	for (char** it = tokens.begin();it != tokens.end(); ++it)
		std::cout << "<" << *it << "> ";
}

void main()
{
	test_tokenizer();
}

#include <string>
#include <assert.h>
#include <iostream>

namespace boko
{
	template <class _CharType>
	class tokenizerT
	{
	public:
	
	    typedef std::basic_string<_CharType> tstring;
	
	    tokenizerT(__in const tstring &str, __in const tstring &token) : m_str(str),
	                                                                    m_token(token),
	                                                                    m_endIter(str, token, -1)
	    {
	    }
	
	    class iterator
	    {
	    public:
	        iterator(__in const tstring &str,
	                 __in const tstring &token,
	                 __in size_t pos = 0) : m_str(str),
	                                        m_token(token),
	                                        m_pos(pos)
	        {
	            m_endpos = tstring::npos;
	
	            if (pos != -1)
	                find_next();
	        }
	
	        const _CharType* operator *()
	        {
	            if (m_pos == -1)
	            {
	                // attempt to deference end iterator
	                assert(false);
	                return NULL;
	            }
	
	            if (!m_value.empty())
	                return m_value.c_str();
	
	            return find_next();
	
	        }
	
	        iterator & operator ++()
	        {
	            if (m_pos == -1)
	                return *this;
	
	            if (m_endpos == -1)
	            {
	                // Now end
	                m_pos = -1;
	                return *this;
	            }
	
	            m_pos = m_endpos;
	            m_endpos = tstring::npos;
	
	            find_next();
	
	            return *this;
	        }
	
	        bool operator != (const iterator &rhs)
	        {
	            if (this->m_pos == -1 && rhs.m_pos == -1)
	                return false;
	            else
	                return this->m_value != rhs.m_value;
	        }

        private:

            const _CharType* find_next() 
            {
                if (m_pos == -1)
                    return NULL;

                // Find the first one
                while (m_pos < m_str.size() && m_token.find(m_str[m_pos]) != tstring::npos)
                    ++m_pos;

                if (m_pos == m_str.size())
                {
                    m_pos = -1; // end iterator
                    return NULL;
                }

                // Find the end pos
                if (m_endpos == tstring::npos)
                {
                    for (size_t i = m_pos; i < m_str.size(); ++i)
                    {
                        if (m_token.find(m_str[i]) != tstring::npos)
                        {
                            m_endpos = i;
                            break;
                        }

                    }  // End for

                    // The end
                    if (m_endpos == tstring::npos)
                        m_endpos = -1;
                }

                if (m_endpos == -1)
                    m_value = m_str.substr(m_pos);
                else
                    m_value = m_str.substr(m_pos, m_endpos - m_pos);

                return m_value.c_str();
            }
	
	    private:
	        const tstring &m_str;
	        const tstring &m_token;
	        size_t      m_pos;
	        size_t      m_endpos;
	        tstring     m_value;
	    };
	
	    iterator begin()
	    {
	        return iterator(m_str, m_token, 0);
	    }
	
	    iterator end()
	    {
	        return m_endIter;
	    }
	
	private:
	    tstring     m_str;
	    tstring     m_token;
	    iterator    m_endIter;
	};

    typedef tokenizerT<char>        tokenizer;
    typedef tokenizerT<wchar_t>     wtokenizer;
}

int _tmain(int argc, _TCHAR* argv[])
{
    std::string str = ";;Hello|world||-foo--bar;yow;baz|";

    boko::tokenizer tokens(str, "-;|");

    for (boko::tokenizer::iterator it = tokens.begin();
        it != tokens.end(); ++it)
        std::cout << "<" << *it << "> ";

    std::cout << "\n";

    std::wstring wstr = L";;Hello|world||-foo--bar;yow;baz|";

    boko::wtokenizer wtokens(wstr, L"-;|");

    for (boko::wtokenizer::iterator it = wtokens.begin();
        it != wtokens.end(); ++it)
        std::wcout << L"<" << *it << L"> ";

	return 0;
}

#include <windows.h>
#include <iostream>
#include <string>

namespace boko {  

  class token_iterator
  {
    typedef std::string::const_iterator Itor;  
    LPCSTR f_;
    Itor begin_;
    Itor end_;

  public:
    token_iterator(LPCSTR f, Itor begin, Itor e)
    : f_(f),begin_(begin),end_(e){ }

    bool operator != (const token_iterator & other) 
    { 
      skip();
      return begin_ != other.begin_;
    }
    void operator ++() 
    { 
      skip();
      Itor p = begin_;
      while (p != end_ && !IsSeperator(*p))
      {
        p++;
      }
      begin_ = p; 
    }
    std::string operator *() 
    { 
      std::string s;

      skip();
      Itor p = begin_;
      while (p != end_ && !IsSeperator(*p))
      {
        s += *p++;
      }
      return s;
    }
  private:
    bool IsSeperator(char c)
    {
      LPCSTR p = f_;

      while (*p != 0)
      {
        if (*p++ == c)
          return true;
      }
      return false;
    }
    void skip()
    {
      Itor p = begin_;
      while (p != end_ && IsSeperator(*p))
        p++;
      begin_ = p;
    }
  };
  
  class tokenizer
  {
    typedef std::string::const_iterator Itor;    
    Itor first_;
    Itor last_;
    LPCSTR f_;
  public:
    typedef token_iterator iterator;
    
    tokenizer(const std::string& s, LPCSTR f)
      : first_(s.begin()), last_(s.end()), f_(f) { }
        
    iterator begin() const { return iterator(f_,first_,last_); }
    iterator end() const { return iterator(f_,last_,last_); }        
  };

} // namespace boko

void test_tokenizer()
{
  std::string str = ";;Hello|world||-foo--bar;yow;baz|";
  
  boko::tokenizer tokens(str, "-;|");

  for (boko::tokenizer::iterator it = tokens.begin();
       it != tokens.end(); ++it)
    std::cout << "<" << *it << "> ";

  // 要求输出 <Hello> <world> <foo> <bar> <yow> <baz>
}