Merge pull request #14 from skadefro/master

 Add support for nested GetElement

OpenRPA.IE/Activities/GetElement.cs

@@ -36,12 +36,13 @@ protected override void Execute(NativeActivityContext context)
             var selector = Selector.Get(context);
             var sel = new IESelector(selector);
             var timeout = TimeSpan.FromSeconds(3);
+            var from = From.Get(context);
             IEElement[] elements = { };
             var sw = new Stopwatch();
             sw.Start();
             do
             {
-                elements = IESelector.GetElementsWithuiSelector(sel, null);
+                elements = IESelector.GetElementsWithuiSelector(sel, from);
             } while (elements .Count() == 0 && sw.Elapsed < timeout);
             context.SetValue(Elements, elements);
             IEnumerator<IEElement> _enum = elements.ToList().GetEnumerator();

OpenRPA.IE/Activities/GetElementDesigner.xaml.cs

@@ -27,17 +27,44 @@ private void NotifyPropertyChanged(String propertyName)
 
         private void Open_Selector(object sender, RoutedEventArgs e)
         {
+            ModelItem loadFrom = ModelItem.Parent;
+            string loadFromSelectorString = "";
+            IESelector anchor = null;
+            while (loadFrom.Parent != null)
+            {
+                var p = loadFrom.Properties.Where(x => x.Name == "Selector").FirstOrDefault();
+                if (p != null)
+                {
+                    loadFromSelectorString = loadFrom.GetValue<string>("Selector");
+                    anchor = new IESelector(loadFromSelectorString);
+                    break;
+                }
+                loadFrom = loadFrom.Parent;
+            }
             string SelectorString = ModelItem.GetValue<string>("Selector");
-            //int maxresult = ModelItem.GetValue<int>("MaxResults");
-            int maxresult = 1;
-
-            if (string.IsNullOrEmpty(SelectorString)) SelectorString = "[]";
-            var selector = new IESelector(SelectorString);
-            var selectors = new Interfaces.Selector.SelectorWindow("IE", selector, maxresult);
-
+            int maxresults = ModelItem.GetValue<int>("MaxResults");
+            Interfaces.Selector.SelectorWindow selectors;
+            if (!string.IsNullOrEmpty(SelectorString))
+            {
+                var selector = new IESelector(SelectorString);
+                selectors = new Interfaces.Selector.SelectorWindow("IE", selector, anchor, maxresults);
+            }
+            else
+            {
+                var selector = new IESelector("[{Selector: 'IE'}]");
+                selectors = new Interfaces.Selector.SelectorWindow("IE", selector, anchor, maxresults);
+            }
             if (selectors.ShowDialog() == true)
             {
                 ModelItem.Properties["Selector"].SetValue(new InArgument<string>() { Expression = new Literal<string>(selectors.vm.json) });
+                if (anchor != null)
+                {
+                    ModelItem.Properties["From"].SetValue(new InArgument<IEElement>()
+                    {
+                        Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue<IEElement>("item")
+                    });
+
+                }
             }
         }
 

OpenRPA.IE/IEElement.cs

@@ -41,6 +41,26 @@ public IEElement(Browser browser, mshtml.IHTMLElement Element)
             }
         }
 
+        public IEElement[] Children
+        {
+            get
+            {
+                var result = new List<IEElement>();
+                mshtml.IHTMLElementCollection children = RawElement.children;
+                foreach (mshtml.IHTMLElement c in children)
+                {
+                    try
+                    {
+                        result.Add(new IEElement(Browser, c));
+                    }
+                    catch (Exception)
+                    {
+                    }
+                }
+                return result.ToArray();
+            }
+        }
+
         private System.Drawing.Rectangle? _Rectangle = null;
         public System.Drawing.Rectangle Rectangle
         {

OpenRPA.IE/IESelector.cs

@@ -62,7 +62,8 @@ private void enumElements(Browser browser, mshtml.IHTMLElement baseelement, IESe
                 var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
                 for (var i = 0; i < anchorlist.Count(); i++)
                 {
-                    if (((IESelectorItem)anchorlist[i]).Match(pathToRoot[0]))
+                    //if (((IESelectorItem)anchorlist[i]).Match(pathToRoot[0]))
+                    if (IESelectorItem.Match(anchorlist[i], pathToRoot[0]))
                     {
                         pathToRoot.Remove(pathToRoot[0]);
                     }
@@ -158,7 +159,15 @@ public override IElement[] GetElements(IElement fromElement = null, int maxresul
         }
         public static IEElement[] GetElementsWithuiSelector(IESelector selector, IElement fromElement = null, int maxresults = 1)
         {
-            var browser = Browser.GetBrowser();
+            IEElement iefromElement = fromElement as IEElement;
+            Browser browser;
+            if (iefromElement != null)
+            {
+                browser = iefromElement.Browser;
+            } else
+            {
+                browser = Browser.GetBrowser();
+            }
             if (browser == null)
             {
                 Log.Warning("Failed locating an Internet Explore instance");
@@ -175,11 +184,20 @@ public static IEElement[] GetElementsWithuiSelector(IESelector selector, IElemen
 
             IEElement[] result = null;
 
-            mshtml.IHTMLElement startfrom = null;
-            if (_fromElement != null) startfrom = _fromElement.RawElement;
-            if (startfrom == null) startfrom = browser.Document.documentElement;
-            current.Add(new IEElement(browser, startfrom));
-            for (var i = 1; i < selectors.Count; i++)
+            int startIndex = 1;
+
+            if(iefromElement != null)
+            {
+                startIndex = 0;
+                current.Add(iefromElement);
+            }
+             else
+            {
+                mshtml.IHTMLElement startfrom = null;
+                startfrom = browser.Document.documentElement;
+                current.Add(new IEElement(browser, startfrom));
+            }
+            for (var i = startIndex; i < selectors.Count; i++)
             {
                 var s = new IESelectorItem(selectors[i]);
                 var elements = new List<IEElement>();

OpenRPA.IE/Plugin.cs

@@ -48,10 +48,15 @@ public treeelement[] GetRootElements(Selector anchor)
         {
             return Plugin._GetRootElements(anchor);
         }
-        public Interfaces.Selector.Selector GetSelector(Interfaces.Selector.treeelement item)
+        public Interfaces.Selector.Selector GetSelector(Selector anchor, Interfaces.Selector.treeelement item)
         {
             var ieitem = item as IETreeElement;
-            return new IESelector(ieitem.IEElement.Browser, ieitem.IEElement.RawElement, null, true, 0, 0);
+            IESelector ieanchor = anchor as IESelector;
+            if (ieanchor == null && anchor != null)
+            {
+                ieanchor = new IESelector(anchor.ToString());
+            }
+            return new IESelector(ieitem.IEElement.Browser, ieitem.IEElement.RawElement, ieanchor, true, 0, 0);
         }
 
         public event Action<IPlugin, IRecordEvent> OnUserAction;

OpenRPA.Interfaces/IPlugin.cs

@@ -17,7 +17,7 @@ public interface IPlugin
         event Action<IPlugin, IRecordEvent> OnUserAction;
         bool parseUserAction(ref IRecordEvent e);
         Selector.treeelement[] GetRootElements(Selector.Selector anchor);
-        Selector.Selector GetSelector(Selector.treeelement item);
+        Selector.Selector GetSelector(Selector.Selector anchor, Selector.treeelement item);
         IElement[] GetElementsWithSelector(Selector.Selector selector, IElement fromElement = null, int maxresults = 1);
         void LaunchBySelector(Selector.Selector selector, TimeSpan timeout);
         bool Match(SelectorItem item, IElement m);

OpenRPA.Interfaces/Selector/SelectorModel.cs

@@ -158,8 +158,11 @@ public void doHighlight()
         public System.Windows.Input.ICommand SelectCommand { get { return new RelayCommand<treeelement>(onSelect); } }
         private void onSelect(treeelement item)
         {
-            var selector = Plugin.GetSelector(item);
+            var selector = Plugin.GetSelector(Anchor, item);
             Selector = selector;
+
+            OnPropertyChanged("Selector");
+            OnPropertyChanged("json");
             // FocusElement(selector);
         }
 

OpenRPA.Java/Activities/GetElement.cs

@@ -32,13 +32,15 @@ protected override void Execute(NativeActivityContext context)
             var sel = new JavaSelector(SelectorString);
             var timeout = TimeSpan.FromSeconds(3);
             var maxresults = MaxResults.Get(context);
+            var from = From.Get(context);
+
             JavaElement[] elements = { };
             var sw = new Stopwatch();
             sw.Start();
             do
             {
                 var selector = new JavaSelector(SelectorString);
-                elements = JavaSelector.GetElementsWithuiSelector(selector, null, maxresults);
+                elements = JavaSelector.GetElementsWithuiSelector(selector, from, maxresults);
 
             } while (elements.Count() == 0 && sw.Elapsed < timeout);
             Log.Debug(string.Format("OpenRPA.Java::GetElement::found {1} elements in {0:mm\\:ss\\.fff}", sw.Elapsed, elements.Count()));

OpenRPA.Java/Activities/GetElementDesigner.xaml.cs

@@ -27,17 +27,45 @@ private void NotifyPropertyChanged(String propertyName)
 
         private void Open_Selector(object sender, RoutedEventArgs e)
         {
+            ModelItem loadFrom = ModelItem.Parent;
+            string loadFromSelectorString = "";
+            JavaSelector anchor = null;
+            while (loadFrom.Parent != null)
+            {
+                var p = loadFrom.Properties.Where(x => x.Name == "Selector").FirstOrDefault();
+                if (p != null)
+                {
+                    loadFromSelectorString = loadFrom.GetValue<string>("Selector");
+                    anchor = new JavaSelector(loadFromSelectorString);
+                    break;
+                }
+                loadFrom = loadFrom.Parent;
+            }
             string SelectorString = ModelItem.GetValue<string>("Selector");
-            //int maxresult = ModelItem.GetValue<int>("MaxResults");
-            int maxresult = 1;
-
-            var selector = new JavaSelector(SelectorString);
-            var selectors = new Interfaces.Selector.SelectorWindow("Java", selector, maxresult);
+            int maxresults = ModelItem.GetValue<int>("MaxResults");
+            Interfaces.Selector.SelectorWindow selectors;
+            if (!string.IsNullOrEmpty(SelectorString))
+            {
+                var selector = new JavaSelector(SelectorString);
+                selectors = new Interfaces.Selector.SelectorWindow("Java", selector, anchor, maxresults);
+            }
+            else
+            {
+                var selector = new JavaSelector("[{Selector: 'Java'}]");
+                selectors = new Interfaces.Selector.SelectorWindow("Java", selector, anchor, maxresults);
+            }
             if (selectors.ShowDialog() == true)
             {
                 ModelItem.Properties["Selector"].SetValue(new InArgument<string>() { Expression = new Literal<string>(selectors.vm.json) });
-            }
+                if (anchor != null)
+                {
+                    ModelItem.Properties["From"].SetValue(new InArgument<JavaElement>()
+                    {
+                        Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue<JavaElement>("item")
+                    });
 
+                }
+            }
         }
 
         private void Highlight_Click(object sender, RoutedEventArgs e)

OpenRPA.Java/JavaSelector.cs

@@ -47,7 +47,8 @@ public JavaSelector(JavaElement element, JavaSelector anchor, bool doEnum)
                 var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
                 for (var i = 0; i < anchorlist.Count; i++)
                 {
-                    if (((JavaSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
+                    //if (((JavaSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
+                    if (JavaSelectorItem.Match(anchorlist[i], pathToRoot[0]))
                     {
                         pathToRoot.Remove(pathToRoot[0]);
                     }
@@ -68,10 +69,14 @@ public JavaSelector(JavaElement element, JavaSelector anchor, bool doEnum)
             element = pathToRoot.Last();
             Clear();
             JavaSelectorItem item;
-            item = new JavaSelectorItem(root, true);
-            item.Enabled = true;
-            item.canDisable = false;
-            Items.Add(item);
+            if (anchor == null)
+            {
+
+                item = new JavaSelectorItem(root, true);
+                item.Enabled = true;
+                item.canDisable = false;
+                Items.Add(item);
+            }
             for (var i = 0; i < pathToRoot.Count(); i++)
             {
                 var o = pathToRoot[i];

OpenRPA.Java/Plugin.cs

@@ -43,10 +43,15 @@ public treeelement[] GetRootElements(Selector anchor)
         {
             return Plugin._GetRootElements(anchor);
         }
-        public Interfaces.Selector.Selector GetSelector(Interfaces.Selector.treeelement item)
+        public Interfaces.Selector.Selector GetSelector(Selector anchor, Interfaces.Selector.treeelement item)
         {
             var javaitem = item as JavaTreeElement;
-            return new JavaSelector(javaitem.JavaElement, null, true);
+            JavaSelector javaanchor = anchor as JavaSelector;
+            if (javaanchor == null && anchor != null)
+            {
+                javaanchor = new JavaSelector(anchor.ToString());
+            }
+            return new JavaSelector(javaitem.JavaElement, javaanchor, true);
         }
         public event Action<IPlugin, IRecordEvent> OnUserAction;
         public string Name { get => "Java"; }

OpenRPA.Net/WebSocketClient.cs

@@ -27,7 +27,15 @@ public class WebSocketClient
 
         public TokenUser user { get; private set; }
         public string jwt { get; private set; }
-
+        public bool isConnected
+        {
+            get
+            {
+                if (ws == null) return false;
+                if(ws.State != WebSocketState.Open) return false;
+                return true;
+            }
+        }
         public WebSocketClient(string url)
         {
             this.url = url;
@@ -37,12 +45,14 @@ public async Task Connect()
             try
             {
                 Log.Debug("Connecting to " + url);
-                if (ws.State == WebSocketState.Aborted || ws.State == WebSocketState.Closed)
+                if (ws != null && (ws.State == WebSocketState.Aborted || ws.State == WebSocketState.Closed))
                 {
                     ws.Dispose();
                     ws = null;
                     ws = new ClientWebSocket();
+                    src = new CancellationTokenSource();
                 }
+                if(ws == null) { ws = new ClientWebSocket();  src = new CancellationTokenSource(); }
                 await ws.ConnectAsync(new Uri(url), src.Token);
                 Log.Information("Connected to " + url);
                 Task receiveTask = Task.Run(async () => await receiveLoop(), src.Token);
@@ -66,8 +76,8 @@ public async Task Close()
                 catch (Exception)
                 {
                 }
-                ws.Dispose();
-                ws = null;
+                //ws.Dispose();
+                //ws = null;
             }
             src.Cancel();
         }
@@ -80,11 +90,14 @@ private async Task receiveLoop()
                 try
                 {
                     if (ws == null) { return; }
-                    if (ws.State != WebSocketState.Open) { return; }
+                    if (ws.State != WebSocketState.Open) {
+                        OnClose?.Invoke("");
+                        return;
+                    }
                     WebSocketReceiveResult result = await ws.ReceiveAsync(new ArraySegment<byte>(buffer), src.Token);
                     json = Encoding.UTF8.GetString(buffer.Take(result.Count).ToArray());
                     var message = JsonConvert.DeserializeObject<SocketMessage>(json);
-                    _receiveQueue.Add(message);
+                    if(message!=null) _receiveQueue.Add(message);
                     await ProcessQueue();
                 }
                 catch (Exception ex)
@@ -104,7 +117,7 @@ private async Task receiveLoop()
         private async Task PingLoop()
         {
             byte[] buffer = new byte[1024];
-            while (true)
+            while (isConnected)
             {
                 await Task.Delay(1000);
                 var msg = new Message("ping");
@@ -118,7 +131,18 @@ public async Task ProcessQueue()
             try
             {
                 await ReceiveSemaphore.WaitAsync();
-                var ids = (from m in _receiveQueue group m by new { m.id } into mygroup select mygroup.Key.id).ToList();
+                if (_receiveQueue == null) return;
+                var tttt = _receiveQueue;
+                List<string> ids = new List<string>();
+                for(var i = 0; i < _receiveQueue.Count; i++)
+                {
+                    if(_receiveQueue[i]!=null)
+                    {
+                        string id = _receiveQueue[i].id;
+                        if (!ids.Contains(id)) ids.Add(id);
+                    }
+                }
+                // ids = (from m in _receiveQueue group m by new { m.id } into mygroup select mygroup.Key.id).ToList();
                 foreach (var id in ids)
                 {
                     var first = _receiveQueue.Where((x) => x.id == id).First();