详解Java豆瓣电影爬虫——小爬虫成长记（附源码）

<dependency>

  <groupid>org.springframework</groupid>

  <artifactid>spring-jdbc</artifactid>

  <version>4.1.1.release</version>

</dependency>

<dependency>

  <groupid>commons-pool</groupid>

  <artifactid>commons-pool</artifactid>

  <version>1.6</version>

</dependency>

<dependency>

  <groupid>commons-dbcp</groupid>

  <artifactid>commons-dbcp</artifactid>

  <version>1.4</version>

</dependency>

<dependency>

  <groupid>mysql</groupid>

  <artifactid>mysql-connector-java</artifactid>

  <version>5.1.38</version>

</dependency>

<dependency>

  <groupid>mysql</groupid>

  <artifactid>mysql-connector-java</artifactid>

  <version>5.1.38</version>

</dependency> 

 <context:property-placeholder location="classpath*:*.properties"/>

<bean id="datasource" class="org.apache.commons.dbcp.basicdatasource" destroy-method="close">

  <property name="driverclassname" value="${jdbc.driver}"/>

  <property name="url" value="${jdbc.url}"/>

  <property name="username" value="${jdbc.username}"/>

  <property name="password" value="${jdbc.password}"/>

</bean> 

<dependency>

  <groupid>org.htmlparser</groupid>

  <artifactid>htmlparser</artifactid>

  <version>1.6</version>

</dependency> 

<dependency>

  <groupid>org.apache.httpcomponents</groupid>

  <artifactid>httpclient</artifactid>

  <version>4.5.2</version>

</dependency>　

<mirrors>

  <mirror>

   <id>alimaven</id>

   <name>aliyun maven</name>

   <url>http://maven.aliyun.com/nexus/content/groups/public/</url>

   <mirrorof>central</mirrorof>    

  </mirror>

</mirrors> 

@test

  public void testfile(){

    file seedfile = new file(this.getclass().getresource("/seed.properties").getpath());

    system.out.print("===========" + seedfile.length() + "===========" );

  }

package java.util.regex;

import java.util.objects;

public final class matcher implements matchresult {

  /**

   * the pattern object that created this matcher.

   */

  pattern parentpattern;

 

  /**

   * the storage used by groups. they may contain invalid values if

   * a group was skipped during the matching.

   */

  int[] groups;

 

  /**

   * the range within the sequence that is to be matched. anchors

   * will match at these "hard" boundaries. changing the region

   * changes these values.

   */

  int from, to;

 

  /**

   * lookbehind uses this value to ensure that the subexpression

   * match ends at the point where the lookbehind was encountered.

   */

  int lookbehindto;

 

  /**

   * the original string being matched.

   */

  charsequence text;

 

  /**

   * matcher state used by the last node. noanchor is used when a

   * match does not have to consume all of the input. endanchor is

   * the mode used for matching all the input.

   */

  static final int endanchor = 1;

  static final int noanchor = 0;

  int acceptmode = noanchor;

 

  /**

   * the range of string that last matched the pattern. if the last

   * match failed then first is -1; last initially holds 0 then it

   * holds the index of the end of the last match (which is where the

   * next search starts).

   */

  int first = -1, last = 0;

 

  /**

   * the end index of what matched in the last match operation.

   */

  int oldlast = -1;

 

  /**

   * the index of the last position appended in a substitution.

   */

  int lastappendposition = 0;

 

  /**

   * storage used by nodes to tell what repetition they are on in

   * a pattern, and where groups begin. the nodes themselves are stateless,

   * so they rely on this field to hold state during a match.

   */

  int[] locals;

 

  /**

   * boolean indicating whether or not more input could change

   * the results of the last match.

   *

   * if hitend is true, and a match was found, then more input

   * might cause a different match to be found.

   * if hitend is true and a match was not found, then more

   * input could cause a match to be found.

   * if hitend is false and a match was found, then more input

   * will not change the match.

   * if hitend is false and a match was not found, then more

   * input will not cause a match to be found.

   */

  boolean hitend;

 

  /**

   * boolean indicating whether or not more input could change

   * a positive match into a negative one.

   *

   * if requireend is true, and a match was found, then more

   * input could cause the match to be lost.

   * if requireend is false and a match was found, then more

   * input might change the match but the match won't be lost.

   * if a match was not found, then requireend has no meaning.

   */

  boolean requireend;

 

  /**

   * if transparentbounds is true then the boundaries of this

   * matcher's region are transparent to lookahead, lookbehind,

   * and boundary matching constructs that try to see beyond them.

   */

  boolean transparentbounds = false;

 

  /**

   * if anchoringbounds is true then the boundaries of this

   * matcher's region match anchors such as ^ and $.

   */

  boolean anchoringbounds = true;

 

  /**

   * no default constructor.

   */

  matcher() {

  }

 

/**

 * all matchers have the state used by pattern during a match.

 */

matcher(pattern parent, charsequence text) {

  this.parentpattern = parent;

  this.text = text;

 

  // allocate state storage

  int parentgroupcount = math.max(parent.capturinggroupcount, 10);

  groups = new int[parentgroupcount * 2];

  locals = new int[parent.localcount];

 

  // put fields into initial states

  reset();

}

....

/**

 * returns the input subsequence matched by the previous match.

 *

 * <p> for a matcher <i>m</i> with input sequence <i>s</i>,

 * the expressions <i>m.</i><tt>group()</tt> and

 * <i>s.</i><tt>substring(</tt><i>m.</i><tt>start(),</tt> <i>m.</i><tt>end())</tt>

 * are equivalent. </p>

 *

 * <p> note that some patterns, for example <tt>a*</tt>, match the empty

 * string. this method will return the empty string when the pattern

 * successfully matches the empty string in the input. </p>

 *

 * @return the (possibly empty) subsequence matched by the previous match,

 *     in string form

 *

 * @throws illegalstateexception

 *     if no match has yet been attempted,

 *     or if the previous match operation failed

 */

public string group() {

  return group(0);

}

/**

 * returns the input subsequence captured by the given group during the

 * previous match operation.

 *

 * <p> for a matcher <i>m</i>, input sequence <i>s</i>, and group index

 * <i>g</i>, the expressions <i>m.</i><tt>group(</tt><i>g</i><tt>)</tt> and

 * <i>s.</i><tt>substring(</tt><i>m.</i><tt>start(</tt><i>g</i><tt>),</tt> <i>m.</i><tt>end(</tt><i>g</i><tt>))</tt>

 * are equivalent. </p>

 *

 * <p> <a href="pattern.html#cg">capturing groups</a> are indexed from left

 * to right, starting at one. group zero denotes the entire pattern, so

 * the expression <tt>m.group(0)</tt> is equivalent to <tt>m.group()</tt>.

 * </p>

 *

 * <p> if the match was successful but the group specified failed to match

 * any part of the input sequence, then <tt>null</tt> is returned. note

 * that some groups, for example <tt>(a*)</tt>, match the empty string.

 * this method will return the empty string when such a group successfully

 * matches the empty string in the input. </p>

 *

 * @param group

 *     the index of a capturing group in this matcher's pattern

 *

 * @return the (possibly empty) subsequence captured by the group

 *     during the previous match, or <tt>null</tt> if the group

 *     failed to match part of the input

 *

 * @throws illegalstateexception

 *     if no match has yet been attempted,

 *     or if the previous match operation failed

 *

 * @throws indexoutofboundsexception

 *     if there is no capturing group in the pattern

 *     with the given index

 */

public string group(int group) {

  if (first < 0)

    throw new illegalstateexception("no match found");

  if (group < 0 || group > groupcount())

    throw new indexoutofboundsexception("no group " + group);

  if ((groups[group*2] == -1) || (groups[group*2+1] == -1))

    return null;

  return getsubsequence(groups[group * 2], groups[group * 2 + 1]).tostring();

}

/**

 * attempts to find the next subsequence of the input sequence that matches

 * the pattern.

 *

 * <p> this method starts at the beginning of this matcher's region, or, if

 * a previous invocation of the method was successful and the matcher has

 * not since been reset, at the first character not matched by the previous

 * match.

 *

 * <p> if the match succeeds then more information can be obtained via the

 * <tt>start</tt>, <tt>end</tt>, and <tt>group</tt> methods. </p>

 *

 * @return <tt>true</tt> if, and only if, a subsequence of the input

 *     sequence matches this matcher's pattern

 */

public boolean find() {

  int nextsearchindex = last;

  if (nextsearchindex == first)

    nextsearchindex++;

 

  // if next search starts before region, start it at region

  if (nextsearchindex < from)

    nextsearchindex = from;

 

  // if next search starts beyond region then it fails

  if (nextsearchindex > to) {

    for (int i = 0; i < groups.length; i++)

      groups[i] = -1;

    return false;

  }

  return search(nextsearchindex);

}

 

/**

 * initiates a search to find a pattern within the given bounds.

 * the groups are filled with default values and the match of the root

 * of the state machine is called. the state machine will hold the state

 * of the match as it proceeds in this matcher.

 *

 * matcher.from is not set here, because it is the "hard" boundary

 * of the start of the search which anchors will set to. the from param

 * is the "soft" boundary of the start of the search, meaning that the

 * regex tries to match at that index but ^ won't match there. subsequent

 * calls to the search methods start at a new "soft" boundary which is

 * the end of the previous match.

 */

boolean search(int from) {

  this.hitend = false;

  this.requireend = false;

  from    = from < 0 ? 0 : from;

  this.first = from;

  this.oldlast = oldlast < 0 ? from : oldlast;

  for (int i = 0; i < groups.length; i++)

    groups[i] = -1;

  acceptmode = noanchor;

  boolean result = parentpattern.root.match(this, from, text);

  if (!result)

    this.first = -1;

  this.oldlast = this.last;

  return result;

}

...

} 

boolean search(int from) {

  this.hitend = false;

  this.requireend = false;

  from    = from < 0 ? 0 : from;

  this.first = from;

  this.oldlast = oldlast < 0 ? from : oldlast;

  for (int i = 0; i < groups.length; i++)

    groups[i] = -1;

  acceptmode = noanchor;

  boolean result = parentpattern.root.match(this, from, text);

  if (!result)

    this.first = -1;

  this.oldlast = this.last;

  return result;

}