1 package net.sourceforge.phpeclipse.wiki.actions.mediawiki.connect;
3 //Parts of this sources are copied and modified from the jEdit Wikipedia plugin:
4 //http://www.djini.de/software/wikipedia/index.html
6 //The modified sources are available under the "Common Public License"
7 //with permission from the original author: Daniel Wunsch
9 import java.io.IOException;
10 import java.io.StringReader;
11 import java.io.UnsupportedEncodingException;
12 import java.net.URLDecoder;
13 import java.util.ArrayList;
14 import java.util.regex.Matcher;
15 import java.util.regex.Pattern;
17 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.config.IWikipedia;
18 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.MethodException;
19 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.PageNotEditableException;
20 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.UnexpectedAnswerException;
21 import net.sourceforge.phpeclipse.wiki.editor.WikiEditorPlugin;
23 import org.apache.commons.httpclient.ConnectMethod;
24 import org.apache.commons.httpclient.HostConfiguration;
25 import org.apache.commons.httpclient.HttpClient;
26 import org.apache.commons.httpclient.HttpConnection;
27 import org.apache.commons.httpclient.HttpException;
28 import org.apache.commons.httpclient.HttpMethod;
29 import org.apache.commons.httpclient.HttpState;
30 import org.apache.commons.httpclient.HttpStatus;
31 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
32 import org.apache.commons.httpclient.NameValuePair;
33 import org.apache.commons.httpclient.URI;
34 import org.apache.commons.httpclient.UsernamePasswordCredentials;
35 import org.apache.commons.httpclient.methods.GetMethod;
36 import org.apache.commons.httpclient.methods.PostMethod;
37 import org.apache.commons.httpclient.protocol.Protocol;
38 import org.apache.commons.httpclient.util.EncodingUtil;
39 import org.eclipse.core.runtime.CoreException;
40 import org.eclipse.core.runtime.Preferences;
43 * This class gets the wikitext from a wikipedia edit page
45 * The basic coding was copied from the commons-httpclient example <code>MediaWikiConnector.java</code>
47 public class MediaWikiConnector {
48 //pattern used to scarp an edit page
49 private static final Pattern BODY_PATTERN = Pattern.compile(
51 * action=".*?title=(.*?)(&|\") <form id="editform" name="editform" method="post"
52 * action="/w/wiki.phtml?title=Ammersee&action=submit" locked pages: <textarea cols='80' rows='25' readonly>
54 ".*<form[^>]*\\sid=\"editform\"[^>]*title=(.*?)&[^>]*>" + ".*<textarea[^>]*\\sname=\"wpTextbox1\"[^>]*>(.*?)</textarea>"
55 + ".*<input[^>]*\\svalue=\"(\\d*)\"[^>]*\\sname=\"wpEdittime\"[^>]*>" + ".*", Pattern.DOTALL);
57 // <input type='hidden' value="53ee6d8b42ff9b7d" name="wpEditToken" />
58 private static final Pattern EDIT_TOKEN = Pattern.compile(
59 ".*<input\\stype='hidden'\\svalue=\"(.*?)\"\\sname=\"wpEditToken\"\\s/>.*", Pattern.DOTALL);
61 //setup default user agent
62 final static public String userAgent = "plog4u.org/0.0";
64 // create a ConnectionManager
65 private MultiThreadedHttpConnectionManager manager;
67 private HttpClient client;
70 * Delay a new store to 1 second
72 private Throttle loadThrottle = new Throttle(WikiEditorPlugin.HTTP_GET_THROTTLE);
75 * Delay a new store to 1 second
77 private Throttle storeThrottle = new Throttle(WikiEditorPlugin.HTTP_PUT_THROTTLE);
80 private long nextTime = 0;
82 private final long minimumDelay;
84 public Throttle(long minimumDelay) {
85 this.minimumDelay = minimumDelay;
88 public Throttle(String key) {
89 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
90 String minimumDelay = prefs.getString(key);
91 this.minimumDelay = Integer.parseInt(minimumDelay);
94 /** this is called from the client */
95 public synchronized void delay() throws InterruptedException {
96 long delay = nextTime - System.currentTimeMillis();
99 nextTime = System.currentTimeMillis() + minimumDelay;
103 public MediaWikiConnector() {
104 // <a href="javascript:window.location.href='http://127.0.0.1:8009/open/?' + window.location.href">bookmarklet</a>
105 manager = new MultiThreadedHttpConnectionManager();
106 manager.setMaxConnectionsPerHost(6);
107 manager.setMaxTotalConnections(18);
108 manager.setConnectionStaleCheckingEnabled(true);
109 // open the conversation
110 client = new HttpClient(manager);
111 setHTTPClientParameters(client);
112 //client.State.CookiePolicy = CookiePolicy.COMPATIBILITY;
113 //client.HostConfiguration.setHost(LOGON_SITE, LOGON_PORT, "http");
116 /** destructor freeing all resources. the Connection is not usable any more after calling this method */
117 public void destroy() {
121 /** log in - returns success */
122 public boolean login(IWikipedia config, String actionUrl, String user, String password, boolean remember)
123 throws UnexpectedAnswerException, MethodException {
124 PostMethod method = new PostMethod(actionUrl);
125 method.setFollowRedirects(false);
126 method.addRequestHeader("User-Agent", userAgent);
127 NameValuePair[] params = new NameValuePair[] {
128 new NameValuePair("title", config.getLoginTitle()),
129 new NameValuePair("action", "submit"),
130 new NameValuePair("wpName", user),
131 new NameValuePair("wpPassword", password),
132 new NameValuePair("wpRemember", remember ? "1" : "0"),
133 new NameValuePair("wpLoginattempt", "submit") };
134 method.addParameters(params);
138 int responseCode = client.executeMethod(method);
139 String responseBody = method.getResponseBodyAsString();
145 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
146 && responseBody.matches(config.getLoginSuccess())) {
148 } else if (responseCode == 200 && responseBody.matches(config.getLoginWrongPw()) || responseCode == 200
149 && responseBody.matches(config.getLoginNoUser())) {
151 if (responseBody.matches(config.getLoginNoUser())) {
152 throw new UnexpectedAnswerException("login not successful: wrong user name: " + user);
153 } else if (responseBody.matches(config.getLoginWrongPw())) {
154 throw new UnexpectedAnswerException("login not successful: wrong password for user: " + user);
156 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
159 throw new UnexpectedAnswerException("login not successful: " + method.getStatusLine());
161 } catch (HttpException e) {
162 throw new MethodException("method failed", e);
163 } catch (IOException e) {
164 throw new MethodException("method failed", e);
166 method.releaseConnection();
169 * // display cookies System.err.println("login: " + result); for (var cookie : client.State.Cookies) {
170 * System.err.println("cookie: " + cookie); }
174 SiteState state = SiteState.siteState(config);
175 state.loggedIn = result;
176 state.userName = user;
181 /** log out - return success */
182 public boolean logout(IWikipedia config, String actionUrl) throws UnexpectedAnswerException, MethodException {
183 GetMethod method = new GetMethod(actionUrl);
184 method.setFollowRedirects(false);
185 method.addRequestHeader("User-Agent", userAgent);
186 NameValuePair[] params = new NameValuePair[] {
187 new NameValuePair("title", config.getLogoutTitle()),
188 new NameValuePair("action", "submit") };
189 method.setQueryString(EncodingUtil.formUrlEncode(params, config.getCharSet()));
193 int responseCode = client.executeMethod(method);
194 String responseBody = method.getResponseBodyAsString();
197 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
198 && responseBody.matches(config.getLogoutSuccess())) {
199 // config.getloggedIn = false;
201 } else if (responseCode == 200) {
202 //### should check for a failure message
204 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
206 throw new UnexpectedAnswerException("logout not successful: " + method.getStatusLine());
208 } catch (HttpException e) {
209 throw new MethodException("method failed", e);
210 } catch (IOException e) {
211 throw new MethodException("method failed", e);
213 method.releaseConnection();
217 SiteState state = SiteState.siteState(config);
218 state.loggedIn = false;
223 /** parses a returned editform for the sessions wpEditToken */
224 private String parseEditToken(String charSet, String responseBody) throws PageNotEditableException {
225 Matcher matcher = EDIT_TOKEN.matcher(responseBody);
226 if (!matcher.matches()) {
230 return matcher.group(1);
234 * returns the edit token or <code>null</code> if no token is available
240 * @throws UnexpectedAnswerException
241 * @throws MethodException
242 * @throws PageNotEditableException
244 public String loadEditToken(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
245 PageNotEditableException {
246 GetMethod method = new GetMethod(actionURL);
247 method.setFollowRedirects(false);
248 method.addRequestHeader("User-Agent", userAgent);
249 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
250 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
253 int responseCode = client.executeMethod(method);
254 String responseBody = method.getResponseBodyAsString();
257 if (responseCode == 200) {
258 String parsed = parseEditToken(charSet, responseBody);
259 if (parsed != null && parsed.length() == 0) {
264 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
266 } catch (HttpException e) {
267 throw new MethodException("method failed", e);
268 } catch (IOException e) {
269 throw new MethodException("method failed", e);
271 method.releaseConnection();
275 /** parses a returned editform into a Content object with UNIX-EOLs ("\n") */
276 private Parsed parseBody(String charSet, String responseBody) throws PageNotEditableException, UnsupportedEncodingException {
277 Matcher matcher = BODY_PATTERN.matcher(responseBody);
278 if (!matcher.matches())
279 throw new PageNotEditableException("cannot find editform form");
281 String title = matcher.group(1);
282 String body = matcher.group(2);
283 String timestamp = matcher.group(3);
284 String tokenEdit = null;
285 // String tokenEdit = matcher.group(4);
286 title = URLDecoder.decode(title, charSet);
287 body = body.replaceAll(""", "\"").replaceAll("'", "'").replaceAll("<", "<").replaceAll(">", ">").replaceAll(
288 "&", "&").replaceAll("\r\n", "\n").replace('\r', '\n');
290 return new Parsed(timestamp, title, body, tokenEdit);
293 /** load a Page Version - returns a Loaded Object */
294 public Loaded load(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
295 PageNotEditableException {
296 GetMethod method = new GetMethod(actionURL);
297 method.setFollowRedirects(false);
298 method.addRequestHeader("User-Agent", userAgent);
299 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
300 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
304 int responseCode = client.executeMethod(method);
305 String responseBody = method.getResponseBodyAsString();
308 if (responseCode == 200) {
309 Parsed parsed = parseBody(charSet, responseBody);
310 Content content = new Content(parsed.timestamp, parsed.body);
311 result = new Loaded(actionURL, charSet, parsed.title, content);
313 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
315 } catch (HttpException e) {
316 throw new MethodException("method failed", e);
317 } catch (IOException e) {
318 throw new MethodException("method failed", e);
320 method.releaseConnection();
325 public String loadHTMLPage(String actionURL, String charSet) throws UnexpectedAnswerException, MethodException,
326 PageNotEditableException {
327 GetMethod method = new GetMethod(actionURL);
328 method.setFollowRedirects(true);
329 method.addRequestHeader("User-Agent", userAgent);
330 // NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title) };
331 // method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
335 int responseCode = client.executeMethod(method);
336 String responseBody = method.getResponseBodyAsString();
339 if (responseCode == 200) {
340 result = responseBody;
341 // Parsed parsed = parseBody(charSet, responseBody);
342 // Content content = new Content(parsed.timestamp, parsed.body);
343 // result = new Loaded(actionURL, charSet, parsed.title, content);
345 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
347 } catch (HttpException e) {
348 throw new MethodException("method failed", e);
349 } catch (IOException e) {
350 throw new MethodException("method failed", e);
352 method.releaseConnection();
357 public ArrayList loadXML(IWikipedia config, String actionURL, String pages) throws UnexpectedAnswerException, MethodException,
358 InterruptedException {
359 loadThrottle.delay();
360 PostMethod method = new PostMethod(actionURL);
361 method.setFollowRedirects(false);
362 method.addRequestHeader("User-Agent", userAgent);
363 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
365 NameValuePair[] params = new NameValuePair[] {
366 new NameValuePair("pages", pages),
367 new NameValuePair("curonly", "X"),
368 new NameValuePair("action", "submit") };
369 method.addParameters(params);
371 int responseCode = client.executeMethod(method);
372 String responseBody = method.getResponseBodyAsString();
374 if (responseCode == 200) {
375 StringReader reader = new StringReader(responseBody);
376 return XMLReader.readFromStream(reader);
378 throw new UnexpectedAnswerException("XML load not successful: expected 200 OK, got " + method.getStatusLine());
380 } catch (CoreException e) {
381 throw new UnexpectedAnswerException("XML load method failed" + e.getMessage());
382 } catch (HttpException e) {
383 throw new MethodException("XML load method failed", e);
384 } catch (IOException e) {
385 throw new MethodException("XML load method failed", e);
387 method.releaseConnection();
392 * store a Page Version - returns a Stored object
395 * WiKipedia predefined properties
403 * @throws UnexpectedAnswerException
404 * @throws MethodException
405 * @throws PageNotEditableException
406 * @throws InterruptedException
408 public Stored store(IWikipedia config, String editToken, String actionUrl, String title, Content content, String summary,
409 boolean minorEdit, boolean watchThis) throws UnexpectedAnswerException, MethodException, PageNotEditableException,
410 InterruptedException {
411 //### workaround: prevent too many stores at a time
412 storeThrottle.delay();
414 PostMethod method = new PostMethod(actionUrl);
416 method.setFollowRedirects(false);
417 method.addRequestHeader("User-Agent", userAgent);
418 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
419 if (editToken == null) {
420 // in some versions editToken isn't supported
423 NameValuePair[] params = new NameValuePair[] {
424 // new NameValuePair("wpSection", ""),
425 // new NameValuePair("wpPreview", "Vorschau zeigen"),
426 // new NameValuePair("wpSave", "Artikel speichern"),
427 new NameValuePair("title", title),
428 new NameValuePair("wpTextbox1", content.body),
429 new NameValuePair("wpEdittime", content.timestamp),
430 new NameValuePair("wpSummary", summary),
431 new NameValuePair("wpEditToken", editToken),
432 new NameValuePair("wpSave", "yes"),
433 new NameValuePair("action", "submit") };
434 method.addParameters(params);
436 method.addParameter("wpMinoredit", "1");
438 method.addParameter("wpWatchthis", "1");
442 int responseCode = client.executeMethod(method);
443 String responseBody = method.getResponseBodyAsString();
446 // since 11dec04 there is a single linefeed instead of an empty page.. trim() helps.
447 if (responseCode == 302 && responseBody.trim().length() == 0) {
448 // log("store successful, reloading");
449 Loaded loaded = load(actionUrl, config.getCharSet(), title);
450 result = new Stored(actionUrl, config.getCharSet(), loaded.title, loaded.content, false);
451 } else if (responseCode == 200) {
452 // log("store not successful, conflict detected");
453 Parsed parsed = parseBody(config.getCharSet(), responseBody);
454 Content cont = new Content(parsed.timestamp, parsed.body);
455 result = new Stored(actionUrl, config.getCharSet(), parsed.title, cont, true);
457 throw new UnexpectedAnswerException("store not successful: expected 200 OK, got " + method.getStatusLine());
459 } catch (HttpException e) {
460 throw new MethodException("method failed", e);
461 } catch (IOException e) {
462 throw new MethodException("method failed", e);
464 method.releaseConnection();
470 * Get the text of a wikimedia article
473 public String getWikiRawText(String wikiname, String urlStr) {
475 // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=raw
476 // http://en.wikibooks.org/w/index.php?title=Programming:PHP:SQL_Injection&action=raw
477 // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=raw
478 HttpMethod method = null;
480 if (urlStr == null) {
481 WikiEditorPlugin.getDefault().reportError("No Wikipedia URL configured", "URL-String == null");
482 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=raw";
484 URI uri = new URI(urlStr.toCharArray());
486 String schema = uri.getScheme();
487 if ((schema == null) || (schema.equals(""))) {
490 Protocol protocol = Protocol.getProtocol(schema);
492 method = new GetMethod(uri.toString());
493 String host = uri.getHost();
494 int port = uri.getPort();
496 HttpConnection connection = new HttpConnection(host, port, protocol);
497 HttpState state = setHTTPParameters(connection);
499 if (connection.isProxied() && connection.isSecure()) {
500 method = new ConnectMethod(method);
503 method.execute(state, connection);
504 // client.executeMethod(method);
506 if (method.getStatusCode() == HttpStatus.SC_OK) {
507 // get the wiki text now:
508 String wikiText = method.getResponseBodyAsString();
511 } catch (Throwable e) {
512 WikiEditorPlugin.log(e);
513 WikiEditorPlugin.getDefault().reportError("Exception occured", e.getMessage() + "\nSee stacktrace in /.metadata/.log file.");
515 if (method != null) {
516 method.releaseConnection();
519 return null; // no success in getting wiki text
522 // public static String getWikiEditTextarea(String wikiname, String urlStr) {
524 // // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=edit
525 // // http://en.wikibooks.org/w/wiki.phtml?title=Programming:PHP:SQL_Injection&action=edit
526 // // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=edit
527 // HttpMethod method = null;
529 // if (urlStr == null) {
530 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=edit";
533 // // urlStr = urlStr + "?title=" + wikiname + "&action=edit";
535 // URI uri = new URI(urlStr.toCharArray());
537 // String schema = uri.getScheme();
538 // if ((schema == null) || (schema.equals(""))) {
541 // Protocol protocol = Protocol.getProtocol(schema);
543 // HttpState state = new HttpState();
545 // method = new GetMethod(uri.toString());
546 // String host = uri.getHost();
547 // int port = uri.getPort();
549 // HttpConnection connection = new HttpConnection(host, port, protocol);
551 // connection.setProxyHost(System.getProperty("http.proxyHost"));
552 // connection.setProxyPort(Integer.parseInt(System.getProperty("http.proxyPort", "80")));
554 // if (System.getProperty("http.proxyUserName") != null) {
555 // state.setProxyCredentials(null, null, new UsernamePasswordCredentials(System.getProperty("http.proxyUserName"), System
556 // .getProperty("http.proxyPassword")));
559 // if (connection.isProxied() && connection.isSecure()) {
560 // method = new ConnectMethod(method);
563 // method.execute(state, connection);
565 // if (method.getStatusCode() == HttpStatus.SC_OK) {
566 // // get the textareas wiki text now:
567 // InputStream stream = method.getResponseBodyAsStream();
568 // int byteLen = stream.available();
570 // byte[] buffer = new byte[byteLen];
571 // stream.read(buffer, 0, byteLen);
572 // String wikiText = new String(buffer);
573 // // String wikiText = method.getResponseBodyAsString();
574 // int start = wikiText.indexOf("<textarea");
575 // if (start != (-1)) {
576 // start = wikiText.indexOf(">", start + 1);
577 // if (start != (-1)) {
578 // int end = wikiText.indexOf("</textarea>");
579 // wikiText = wikiText.substring(start + 1, end);
583 // // System.out.println(wikiText);
586 // } catch (Exception e) {
587 // e.printStackTrace();
589 // if (method != null) {
590 // method.releaseConnection();
593 // return null; // no success in getting wiki text
600 private HttpState setHTTPParameters(HttpConnection connection) {
601 HttpState state = new HttpState();
602 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
603 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
604 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
607 // timeout after xx seconds
608 connection.setConnectionTimeout(Integer.parseInt(timeout));
610 if (proxyHost.length() > 0) {
611 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
612 connection.setProxyHost(proxyHost);
613 connection.setProxyPort(Integer.parseInt(proxyPort));
615 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
616 if (proxyUserName.length() > 0) {
617 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
618 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
622 } catch (Exception e) {
628 private void setHTTPClientParameters(HttpClient client) {
630 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
631 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
632 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
635 // timeout after xx seconds
636 client.setConnectionTimeout(Integer.parseInt(timeout));
638 if (proxyHost.length() > 0) {
639 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
640 HostConfiguration conf = new HostConfiguration();
641 client.setHostConfiguration(conf);
642 conf.setProxy(proxyHost, Integer.parseInt(proxyPort));
644 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
645 if (proxyUserName.length() > 0) {
646 HttpState state = new HttpState();
647 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
648 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
649 client.setState(state);
653 } catch (Exception e) {
659 public static void main(String[] args) {
660 MediaWikiConnector mwc = new MediaWikiConnector();
662 IWikipedia wp = null;
663 ArrayList list = mwc.loadXML(wp, "http://www.plog4u.de/wiki/index.php/Spezial:Export", "Mechanisches Fernsehen\nSynästhesie");
664 for (int i = 0; i < list.size(); i++) {
665 System.out.println(list.get(i).toString());
667 } catch (UnexpectedAnswerException e) {
668 // TODO Auto-generated catch block
670 } catch (Exception e) {
671 // TODO Auto-generated catch block